def new_post(blog_id, username, password, post, publish):
"""metaWeblog.newPost(blog_id, username, password, post, publish)
=> post_id"""
user = authenticate(username, password)
if post.get('dateCreated'):
pub_date = datetime.strptime(
post['dateCreated'].value.replace('Z', '').replace('-', ''),
'%Y%m%dT%H:%M:%S')
else:
pub_date = datetime.now()
entry_dict = {'title': post['title'],
'body': safe(removetags(post['description'],HTML_TAGS)),
'html_body': safe(post['description']),
'pub_date': pub_date,
'published': 1,
'last_update': pub_date,
'comment_enabled': post.get('mt_allow_comments', 1) == 1,
'slug': slugify(post['title']),
}
entry = Post.objects.create(**entry_dict)
if 'categories' in post:
entry.categories.add(*[Category.objects.get_or_create(
title=cat, slug=slugify(cat))[0]
for cat in post['categories']])
if 'tags' in post:
entry.tags.add(*[Tag.objects.get_or_create(
title=tag, slug=slugify(tag))[0]
for tag in post['tags']])
return entry.pk
def pull_html_from_email_file(fname):
content = open(fname, 'r').read()
idx = content.find('<html')
if idx == -1:
return None
end_idx = content.find('</html>', idx + 5)
if end_idx == -1:
return None
html_content = content[idx:end_idx + 7]
# remove line breaks
strings_to_remove = ['=\r\n', '<o:p>', '</o:p>']
for to_remove in strings_to_remove:
html_content = html_content.replace(to_remove, '')
# remove <img ...> and <span...> tags
html_content = removetags(html_content, 'p b img span')
# remove extra attributes
safe_attrs = clean.defs.safe_attrs
clean.defs.safe_attrs = frozenset()
cleaner = clean.Cleaner(safe_attrs_only=True)
html_content = cleaner.clean_html(html_content)
return html_content
def disarm_user_input(html):
"""
Returns html without posible harm
In addition
- urlizes text if no links are used
- breaks lines if no paragraphs are used
"""
html = defaultfilters.removetags(html, "script style comment")
# remove javascript events and style attributes from tags
re_comments = re.compile(r'<!--[\s\S]*?(-->|$)')
re_tags = re.compile(r'(<[^>\s]+)([^>]+)(>)')
re_attrs = re.compile(
r"""\s+(on[^=]+|style)=([^"'\s]+|"[^"]*"|'[^']*')""",
)
def remove_js_events(match):
return "".join((
match.group(1),
re_attrs.sub('', match.group(2)),
match.group(3),
))
html = re_comments.sub("", html)
html = re_tags.sub(remove_js_events, html)
if "</a>" not in html:
html = defaultfilters.urlizetrunc(html, "30")
if "</p>" not in html:
html = defaultfilters.linebreaks(html)
html = defaultfilters.safe(html)
return html
def test_removetags(self):
self.assertEqual(removetags('some <b>html</b> with <script>alert'
'("You smell")</script> disallowed <img /> tags', 'script img'),
'some <b>html</b> with alert("You smell") disallowed tags')
self.assertEqual(striptags('some <b>html</b> with <script>alert'
'("You smell")</script> disallowed <img /> tags'),
'some html with alert("You smell") disallowed tags')
def test_non_string_input(self):
# Filters shouldn't break if passed non-strings
self.assertEqual(addslashes(123), '123')
self.assertEqual(linenumbers(123), '1. 123')
self.assertEqual(lower(123), '123')
self.assertEqual(make_list(123), ['1', '2', '3'])
self.assertEqual(slugify(123), '123')
self.assertEqual(title(123), '123')
self.assertEqual(truncatewords(123, 2), '123')
self.assertEqual(upper(123), '123')
self.assertEqual(urlencode(123), '123')
self.assertEqual(urlize(123), '123')
self.assertEqual(urlizetrunc(123, 1), '123')
self.assertEqual(wordcount(123), 1)
self.assertEqual(wordwrap(123, 2), '123')
self.assertEqual(ljust('123', 4), '123 ')
self.assertEqual(rjust('123', 4), ' 123')
self.assertEqual(center('123', 5), ' 123 ')
self.assertEqual(center('123', 6), ' 123 ')
self.assertEqual(cut(123, '2'), '13')
self.assertEqual(escape(123), '123')
self.assertEqual(linebreaks_filter(123), '<p>123</p>')
self.assertEqual(linebreaksbr(123), '123')
self.assertEqual(removetags(123, 'a'), '123')
self.assertEqual(striptags(123), '123')
def test_removetags(self):
self.assertEqual(removetags('some <b>html</b> with <script>alert'
'("You smell")</script> disallowed <img /> tags', 'script img'),
'some <b>html</b> with alert("You smell") disallowed tags')
self.assertEqual(striptags('some <b>html</b> with <script>alert'
'("You smell")</script> disallowed <img /> tags'),
'some html with alert("You smell") disallowed tags')
def edit_post(post_id, username, password, post, publish):
"""metaWeblog.editPost(post_id, username, password, post, publish)
=> boolean"""
user = authenticate(username, password, 'zinnia.change_entry')
entry = Post.objects.get(id=post_id)
if post.get('dateCreated'):
pub_date = datetime.strptime(
post['dateCreated'].value.replace('Z', '').replace('-', ''),
'%Y%m%dT%H:%M:%S')
else:
pub_date = entry.creation_date
entry.title = post['title']
entry.html_body = post['description']
entry.body = safe(removetags(entry.html_body,HTML_TAGS))
entry.pub_date = pub_date
entry.last_update = datetime.datetime.now()
entry.comment_enabled = post.get('mt_allow_comments', 1) == 1
entry.slug = 'wp_slug' in post and post['wp_slug'] or slugify(
post['title'])
entry.save()
if 'categories' in post:
entry.categories.clear()
entry.categories.add(*[Category.objects.get_or_create(
title=cat, slug=slugify(cat))[0]
for cat in post['categories']])
if 'tags' in post:
entry.tags.add(*[Tag.objects.get_or_create(
title=tag, slug=slugify(tag))[0]
for tag in post['tags']])
return True
def pull_html_from_email_file(fname):
content = open(fname, 'r').read()
idx = content.find('<html')
if idx==-1:
return None
end_idx = content.find('</html>', idx+5)
if end_idx==-1:
return None
html_content = content[idx:end_idx+7]
# remove line breaks
strings_to_remove = ['=\r\n', '<o:p>', '</o:p>']
for to_remove in strings_to_remove:
html_content = html_content.replace(to_remove, '')
# remove <img ...> and <span...> tags
html_content = removetags(html_content, 'p b img span')
# remove extra attributes
safe_attrs=clean.defs.safe_attrs
clean.defs.safe_attrs=frozenset()
cleaner = clean.Cleaner(safe_attrs_only=True)
html_content = cleaner.clean_html(html_content)
return html_content
def test_non_string_input(self):
# Filters shouldn't break if passed non-strings
self.assertEqual(addslashes(123), '123')
self.assertEqual(linenumbers(123), '1. 123')
self.assertEqual(lower(123), '123')
self.assertEqual(make_list(123), ['1', '2', '3'])
self.assertEqual(slugify(123), '123')
self.assertEqual(title(123), '123')
self.assertEqual(truncatewords(123, 2), '123')
self.assertEqual(upper(123), '123')
self.assertEqual(urlencode(123), '123')
self.assertEqual(urlize(123), '123')
self.assertEqual(urlizetrunc(123, 1), '123')
self.assertEqual(wordcount(123), 1)
self.assertEqual(wordwrap(123, 2), '123')
self.assertEqual(ljust('123', 4), '123 ')
self.assertEqual(rjust('123', 4), ' 123')
self.assertEqual(center('123', 5), ' 123 ')
self.assertEqual(center('123', 6), ' 123 ')
self.assertEqual(cut(123, '2'), '13')
self.assertEqual(escape(123), '123')
self.assertEqual(linebreaks_filter(123), '<p>123</p>')
self.assertEqual(linebreaksbr(123), '123')
self.assertEqual(removetags(123, 'a'), '123')
self.assertEqual(striptags(123), '123')
def parse_email_body(self, body):
"Removes all the dangerous and useless staff"
# Replace annoying characters
body = body.replace('\r', '').replace('=\n', '').replace('=\n\r', '')
body = body.replace('=20\n', '\n\n')
HARDTREE_MESSAGING_UNSAFE_BLOCKS = getattr(settings, 'HARDTREE_MESSAGING_UNSAFE_BLOCKS',
('head', 'object', 'embed', 'applet', 'noframes',
'noscript', 'noembed', 'iframe', 'frame', 'frameset'))
# Strip unsafe tags
tags_str = ' '.join(HARDTREE_MESSAGING_UNSAFE_BLOCKS)
body = removetags(body, tags_str)
# Remove multiple <br /> tags
rules = [
{r'\s*<br\s*/?>\s*': u'\n'}
]
for rule in rules:
for (k, v) in rule.items():
regex = re.compile(k)
body = regex.sub(v, body)
# displaying messages correctly
if body.startswith('<!DOCTYPE') or body.startswith('<html') or body.startswith('<meta'):
body = body.replace('\n\n\n', '\n').replace(
'\n\n', '\n').replace('>\n<', '><')
elif not '<html' in body:
body = body.replace('\n', '<br />\n')
return body
def strip_tags(process_string, unsafe_tags=None, safe_tags=None):
'''django's removetags only accepts space separated lists of tags (rawr), so we're taking our list and composing that string, while accepting some safe tags to keep
copied from EC code'''
if (not unsafe_tags and not safe_tags
): # if we don't have any args passed or they are invalid
#print "neither"
unsafe_tags = all_tags
elif safe_tags:
#print "safe"
unsafe_tags = list(
set(all_tags) - set(safe_tags)
) # if we provided safe tags, subtract them from the set of all_tags
if unsafe_tags: # we don't need anything else done, but if both args are provided, we should print a warning noting that only the safe is being used since it's more restrictive
pass # TODO: PRINT WARNING
elif unsafe_tags is not None and type(
unsafe_tags
) != list: # this should check for functionality and not type, but for now this is fine
#print "wrong - %s" % type(unsafe_tags)
unsafe_tags = all_tags # start with all tags as the base to strip
elif unsafe_tags:
#print "unsafe"
pass # it's already defined, but we don't want it caught in the else
else:
#print "other"
unsafe_tags = all_tags
remove_str = ""
for item in unsafe_tags:
remove_str += "%s " % item
return removetags(process_string, remove_str)
def parse_email_body(self, body):
"Removes all the dangerous and useless staff"
# Replace annoying characters
body = body.replace('\r', '').replace('=\n', '').replace('=\n\r', '')
body = body.replace('=20\n', '\n\n')
ANAF_MESSAGING_UNSAFE_BLOCKS = settings.ANAF_MESSAGING_UNSAFE_BLOCKS
# Strip unsafe tags
tags_str = ' '.join(ANAF_MESSAGING_UNSAFE_BLOCKS)
body = removetags(body, tags_str)
# Remove multiple <br /> tags
rules = [
{r'\s*<br\s*/?>\s*': u'\n'}
]
for rule in rules:
for (k, v) in rule.items():
regex = re.compile(k)
body = regex.sub(v, body)
# displaying messages correctly
if body.startswith('<!DOCTYPE') or body.startswith('<html') or body.startswith('<meta'):
body = body.replace('\n\n\n', '\n').replace(
'\n\n', '\n').replace('>\n<', '><')
elif '<html' not in body:
body = body.replace('\n', '<br />\n')
return body
def pull_html_from_email_content(email_content):
if email_content is None:
return None
idx = email_content.find('<html')
if idx == -1:
return None
end_idx = email_content.find('</html>', idx + 5)
if end_idx == -1:
return None
html_content = email_content[idx:end_idx + 7]
# remove line breaks
strings_to_remove = ['=\r\n', '<o:p>', '</o:p>']
for to_remove in strings_to_remove:
html_content = html_content.replace(to_remove, '')
# remove <img ...> and <span...> tags
html_content = removetags(html_content, 'p b img span')
# remove extra attributes
safe_attrs = clean.defs.safe_attrs
clean.defs.safe_attrs = frozenset()
cleaner = clean.Cleaner(safe_attrs_only=True)
html_content = cleaner.clean_html(html_content)
html_content = html_content.encode('ascii', 'ignore')
return html_content
def parse_email_body(self, body):
"Removes all the dangerous and useless staff"
# Replace annoying characters
body = body.replace('\r', '').replace('=\n', '').replace('=\n\r', '')
body = body.replace('=20\n', '\n\n')
HARDTREE_MESSAGING_UNSAFE_BLOCKS = getattr(
settings, 'HARDTREE_MESSAGING_UNSAFE_BLOCKS',
('head', 'object', 'embed', 'applet', 'noframes', 'noscript',
'noembed', 'iframe', 'frame', 'frameset'))
# Strip unsafe tags
tags_str = ' '.join(HARDTREE_MESSAGING_UNSAFE_BLOCKS)
body = removetags(body, tags_str)
# Remove multiple <br /> tags
rules = [{r'\s*<br\s*/?>\s*': u'\n'}]
for rule in rules:
for (k, v) in rule.items():
regex = re.compile(k)
body = regex.sub(v, body)
# displaying messages correctly
if body.startswith('<!DOCTYPE') or body.startswith(
'<html') or body.startswith('<meta'):
body = body.replace('\n\n\n',
'\n').replace('\n\n',
'\n').replace('>\n<', '><')
elif not '<html' in body:
body = body.replace('\n', '<br />\n')
return body
def pull_html_from_email_content(email_content):
if email_content is None:
return None
idx = email_content.find('<html')
if idx==-1:
return None
end_idx = email_content.find('</html>', idx+5)
if end_idx==-1:
return None
html_content = email_content[idx:end_idx+7]
# remove line breaks
strings_to_remove = ['=\r\n', '<o:p>', '</o:p>']
for to_remove in strings_to_remove:
html_content = html_content.replace(to_remove, '')
# remove <img ...> and <span...> tags
html_content = removetags(html_content, 'p b img span')
# remove extra attributes
safe_attrs=clean.defs.safe_attrs
clean.defs.safe_attrs=frozenset()
cleaner = clean.Cleaner(safe_attrs_only=True)
html_content = cleaner.clean_html(html_content)
html_content = html_content.encode('ascii', 'ignore')
return html_content
def test_removetags(self):
with warnings.catch_warnings(record=True):
warnings.simplefilter("always")
self.assertEqual(
removetags(
'some <b>html</b> with <script>alert'
'("You smell")</script> disallowed <img /> tags',
'script img'),
'some <b>html</b> with alert("You smell") disallowed tags')
def test_removetags(self):
with warnings.catch_warnings():
warnings.simplefilter('ignore', RemovedInDjango20Warning)
self.assertEqual(
removetags(
'some <b>html</b> with <script>alert("You smell")</script> disallowed <img /> tags',
'script img',
),
'some <b>html</b> with alert("You smell") disallowed tags',
)
def search_cantfind(request):
newcantfind_form = NewCantFindForm(request.GET)
sendnewcantfind_form = SendNewCantFindForm(request.POST or request.GET)
if sendnewcantfind_form.is_valid():
x = sendnewcantfind_form.cleaned_data['x']
y = sendnewcantfind_form.cleaned_data['y']
distance = sendnewcantfind_form.cleaned_data['distance']
ambiance = sendnewcantfind_form.cleaned_data['ambiance']
if request.POST.get('submit', None):
address = reverse_geocode(y, x)
text = render_to_string(template_name='venue/search_cantfind_text.html',
dictionary={'address': address,
'distance': round(distance / 1000.0, 1),
'sendnewcantfind_form': sendnewcantfind_form,
'ambiance': ambiance,
'user': request.user})
send_mail(_("Une nouvelle mission pour SuperLaurent"),
removetags(text, "input strong textarea label p"),
'*****@*****.**',
[settings.VENUE_CANTFIND_EMAIL],
fail_silently=False)
messages.success(request, _("A new mission was successfully sent to SuperLaurent !"))
return redirect('venue:search')
if newcantfind_form.is_valid():
x = newcantfind_form.cleaned_data['x']
y = newcantfind_form.cleaned_data['y']
distance = newcantfind_form.cleaned_data['distance']
ambiance = newcantfind_form.cleaned_data['ambiance']
address = reverse_geocode(y, x)
text = render_to_string(template_name='venue/search_cantfind_text.html',
dictionary={'address': address,
'sendnewcantfind_form': sendnewcantfind_form,
'distance': round(distance / 1000.0, 1),
'ambiance': ambiance,
'user': request.user})
else:
text = _("Type your text here !")
return render_to_response(template_name='venue/search_cantfind.html',
dictionary={'text': text,
'sendnewcantfind_form': sendnewcantfind_form},
context_instance=RequestContext(request)
)
def save(self, *args, **kwargs):
# Clean the body, removing all html tags.
self.body = safe(removetags(self.html_body, HTML_TAGS))
# TO DO #
# Replace the <attach id="slug"> html tag with proper tags:
# for example, <attach id="image-people"> will be replaced with
# '<img src="%s"' />' % reverse('media/slug', [], {'slug':"image-people"})
# so in the html it displays properly.
# Then call the *real* save method.
super(Post, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
# Clean the body, removing all html tags.
self.body = safe(removetags(self.html_body, HTML_TAGS))
# TO DO #
# Replace the <attach id="slug"> html tag with proper tags:
# for example, <attach id="image-people"> will be replaced with
# '<img src="%s"' />' % reverse('media/slug', [], {'slug':"image-people"})
# so in the html it displays properly.
# Then call the *real* save method.
super(Post, self).save(*args, **kwargs)
def get(self, request, instance_id, *args, **kwargs):
self.authenticate(request)
tasks = Task.objects.select_related('choice_set').all()
tasks_list = self.serializer.serialize(tasks)
for task in tasks_list:
choices = Choice.objects.filter(task=task['pk'])
choice_list = self.flatten(self.serializer.serialize(choices))
task['fields']['description'] = removetags(task['fields']['description'], 'span')
task['fields']['choices'] = choice_list
task['fields']['number'] = task['fields']['number'] + 1
return HttpResponse(self.to_json(tasks_list), mimetype='application/json', status=200)
def censor(value, replace=None):
"""
Strips <a> elements, and replace links with blocks or a given strings.
Accepts a replacement string as an optional parameter. If not specified, it will use the SOPA_REPLACE setting (defaults to 'BLOCKS').
The special value 'BLOCKS' will replace every char in the link with a black square.
"""
value = removetags(value, 'a')
if replace is None:
replace = settings.SOPA_REPLACE
if replace == 'BLOCKS':
return url_regex.sub(repl_blocks, value)
return url_regex.sub(replace, value)
def get(self):
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
item_keys = Tip.all(keys_only=True).filter('tweeted =', False).fetch(2000)
if len(item_keys) > 0:
random_key = random.choice(item_keys)
item = db.get(random_key)
content = defaultfilters.removetags(item.content,"a code")
api.update_status(content)
item.tweeted = True
item.put()
def on_post_was_submit(sender,post,*args,**kwargs):
try:
from blog.models import OptionSet
bind = OptionSet.get('bind_weibo','')=='True'
if bind:
access_token_key = OptionSet.get('weibo_access_token_key','')
access_token_secret = OptionSet.get('weibo_access_token_secret','')
weibo_client.setToken(access_token_key,access_token_secret)
from weibopy.api import API
from django.template.defaultfilters import removetags
api = API(weibo_client)
api.update_status(status='[%s] %s ... %s'\
%(post.title,removetags(post.content[:60],'a p span div img br'),\
settings.BLOG_DOMAIN+post.get_absolute_url()))
except:
pass
def get(self, request, instance_id, *args, **kwargs):
self.authenticate(request)
tasks = Task.objects.select_related('choice_set').all()
tasks_list = self.serializer.serialize(tasks)
for task in tasks_list:
choices = Choice.objects.filter(task=task['pk'])
choice_list = self.flatten(self.serializer.serialize(choices))
task['fields']['description'] = removetags(
task['fields']['description'], 'span')
task['fields']['choices'] = choice_list
task['fields']['number'] = task['fields']['number'] + 1
return HttpResponse(self.to_json(tasks_list),
mimetype='application/json',
status=200)
def on_post_was_submit(sender, post, *args, **kwargs):
try:
from blog.models import OptionSet
bind = OptionSet.get('bind_weibo', '') == 'True'
if bind:
access_token_key = OptionSet.get('weibo_access_token_key', '')
access_token_secret = OptionSet.get('weibo_access_token_secret',
'')
weibo_client.setToken(access_token_key, access_token_secret)
from weibopy.api import API
from django.template.defaultfilters import removetags
api = API(weibo_client)
api.update_status(status='[%s] %s ... %s'\
%(post.title,removetags(post.content[:60],'a p span div img br'),\
settings.BLOG_DOMAIN+post.get_absolute_url()))
except:
pass
def notify_if_not_duplicate(request, item):
the_message_content = mark_safe("""<strong><a title='{dismiss_title}'
href='{dismiss_url}'>×</a></strong>
<small><em>{date}</em></small>,
<strong>{created_by}</strong>:
{message}""".format(
created_by=item.created_by,
date=naturalday(item.date),
message=linebreaksbr(removetags(item.content, 'script')),
dismiss_title=_('Dismiss'),
dismiss_url=item.get_delete_url()))
jot_messages = [msg for msg in messages.get_messages(request) \
if msg.level == JOT_MESSAGE_LEVEL]
in_jot_messages = [msg for msg in jot_messages \
if msg.message == the_message_content]
if not in_jot_messages:
messages.add_message(request,
JOT_MESSAGE_LEVEL,
the_message_content,
fail_silently=True)
def remove_img_href(data_soup):
"""
removes <a href> tags that link to images
@param data_soup expects a soup element
@return soup element without hrefs pointing to images
"""
from django.template.defaultfilters import removetags
for a in data_soup.findAll("a"):
# check the url / href of the a tag
# get the filename, if the extension is an image, remove it!
if a.has_key("href"):
href = a["href"]
filename = href.split("/")[-1]
try:
type, mime = get_media_mime_type(filename)
if type == "image":
clean_a = removetags(a, "a")
a.replaceWith(clean_a)
except ContifyValidationError, e:
logger.warn("non image, ignore. Error %s" % (e))
else:
logger.warn("No href in the a tag!! %s" %
(smart_unicode(a.renderContents())))
def clean_description(self):
description = self.cleaned_data.get('description')
return removetags(description, settings.FILTER_HTML_TAGS)
def safe_summary(self):
"""suitable for use in places that must avoid nested anchor tags"""
return removetags(self.summary, "a")
def safe_summary(self):
'''suitable for use in places that must avoid nested anchor tags'''
return removetags(self.summary, 'a')
def test_non_string_input(self):
with warnings.catch_warnings():
warnings.simplefilter('ignore', RemovedInDjango20Warning)
self.assertEqual(removetags(123, 'a'), '123')
def test_non_string_input(self):
self.assertEqual(removetags(123, 'a'), '123')
def safe_summary(self):
'''suitable for use in places that must avoid nested anchor tags'''
return removetags(self.summary, 'a')
def setx(request, name, msg, audit=False):
try:
d = request.session['messages']
if d:
Message.content_msj = request.session['messages']
except KeyError:
pass
Message.content_msj.append(
(u''
u'<div class="alert alert-block alert-%s"><button type="button" class="close" data-dismiss="alert">×</button>%s</div>'
u'' % (name, msg))
)
request.session['messages'] = Message.content_msj
#m = __import__ ('logger')
if audit:
methodToCall = getattr(logger, name) #logger.debug donde name=debug p.e.
methodToCall(("[%s][%s][%s] %s ") % (request.get_full_path(), request.user, request.META['REMOTE_ADDR'], removetags(msg,'b') ) ) #logger.debug(msg) #(str(u'%s'%msg)).encode('utf-8')
def parse_content_html(self, parse_url):
log('parse', parse_url)
urls = []
data = {}
data_string = self.opener.open(parse_url).read()
soup = BeautifulSoup(data_string, "lxml")
if not self.config.article_url_regexp or self.config.article_url_regexp.findall(parse_url):
data = {'shop_url': parse_url, 'shop': self.config.shop_name}
for k, v in self.config.field_match.items():
v, use_html = Parser.use_pseudo(v, '::html')
v, use_href = Parser.use_pseudo(v, '::href')
v, use_all = Parser.use_pseudo(v, '::all')
v, custom_attr = Parser.use_pseudo_with_arg(v, '::attr')
def get_attr(tag, attr):
return tag.attrs.get(custom_attr if custom_attr else attr, '')
hits = soup.select(v)
if len(hits):
tag = hits[0]
if tag.name == 'img':
data[k] = urlparse.urljoin(self.config.base_url, get_attr(tag, 'src'))
elif use_href:
data[k] = get_attr(tag, 'href')
elif use_html:
data[k] = removetags(Parser.purify_text('%s' % tag), 'img script style span br')
elif use_all:
data[k] = [Parser.purify_text(h.text.replace(',', ' ')) for h in hits if
self.valid_tag(h.text)]
if k == 'tags' and self.config.generic_tag:
data[k].append(self.config.generic_tag)
else:
text = get_attr(tag, 'content')
if not text:
text = tag.text
if not text:
text = tag.attrs.get('href', '')
if k == 'image':
text = urlparse.urljoin(self.config.base_url, text)
data[k] = Parser.purify_text(text)
if not data.has_key('title') or not data.has_key('description') or not data.has_key('price'):
data = {}
if not self.config.leaf_only:
# respect <base href="http://www.base.com/shop/"/> head tag
sanitize_url_kwargs = {}
base = soup.select('base')
if len(base):
sanitize_url_kwargs['base'] = base[0].attrs.get('href')
for tag in soup.select(self.config.link_selector):
href = tag.attrs.get('href')
if href and self.is_valid_href(href):
href = self.sanitize_url(href, **sanitize_url_kwargs)
urls.append(href)
soup.decompose()
del soup
del data_string
return data, urls
def test_non_string_input(self):
self.assertEqual(removetags(123, 'a'), '123')
def test_removetags(self):
with warnings.catch_warnings(record=True):
warnings.simplefilter("always")
self.assertEqual(removetags('some <b>html</b> with <script>alert'
'("You smell")</script> disallowed <img /> tags', 'script img'),
'some <b>html</b> with alert("You smell") disallowed tags')
def clean_html(text):
outstr = removetags(text, REMOVE_TAGS)
for attr in REMOVE_ATTRIBUTES:
for rm in re.findall(' '+attr+'=".*?"', outstr):
outstr = outstr.replace(rm,'')
return outstr
def markup(value, parser):
esc = conditional_escape
if parser == 1:
import postmarkup
from pygments import highlight
from pygments.lexers import get_lexer_by_name, ClassNotFound
#from pygments.lexers import guess_lexer
from pygments.formatters import HtmlFormatter
markup = postmarkup.create(annotate_links=False,exclude=["img", "code"],use_pygments=False)
class ImgTag(postmarkup.TagBase):
valid_params = ("left", "right")
def __init__(self, name, **kwargs):
postmarkup.TagBase.__init__(self, name, inline=True)
def render_open(self, parser, node_index):
contents = self.get_contents(parser)
self.skip_contents(parser)
contents = postmarkup.strip_bbcode(contents).replace(u'"', "%22")
if self.params in self.valid_params:
return u'<img class="float-%s" src="%s" alt="%s">' % (self.params, contents, contents)
else:
return u'<img src="%s" alt="%s">' % (contents, contents)
class PygmentsCodeTag(postmarkup.TagBase):
def __init__(self, name, pygments_line_numbers=True, **kwargs):
postmarkup.TagBase.__init__(self, name, enclosed=True, strip_first_newline=True)
self.line_numbers = pygments_line_numbers
def render_open(self, parser, node_index):
contents = self.get_contents(parser)
self.skip_contents(parser)
#if self.params:
try:
lexer = get_lexer_by_name(self.params, stripall=True)
except ClassNotFound:
contents = postmarkup._escape_no_breaks(contents)
return '<div class="code"><pre>%s</pre></div>' % contents
#Well, do we realy need lexer gues?
#else:
# lexer = guess_lexer(contents)
formatter = HtmlFormatter(linenos='inline', cssclass="code")
return highlight(contents, lexer, formatter)
markup.add_tag(ImgTag, u'img')
markup.add_tag(PygmentsCodeTag, u'code')
markup.add_tag(postmarkup.SimpleTag, u'block', u'div class="mblock"')
value = "<p>" + markup(value) + "</p>"
return value
elif parser == 2:
from markdown import markdown
value = esc(value)
value = markdown(value)
return value
elif parser == 3:
from wikimarkup import parselite
value = parselite(value)
elif parser == 4:
from django.template.defaultfilters import removetags
value = removetags(value, 'style html script applet form frame iframe map noframes noscript object var area input button select')
value = linebreaks(value)
else:
value = esc(value)
value = urlize(value)
value = linebreaks(value)
return value
def clean_description(self):
description = self.cleaned_data.get('description')
return removetags(description, settings.FILTER_HTML_TAGS)
def prepare_question(question_details):
'''
1. check whether question title has media files or not.
2. check whether question content has media files or not.
3. prepare content first.
4. prepare media second.
widgets : {
'sound':'',
'image':'',
'video':''
}
'''
question_widgets = {
'widgets':{
'sounds':{},
'images':{},
'videos':{},
}
}
old_question = question_details
old_question_fields = old_question['fields']
old_question_title = old_question_fields['title'].replace('<p>',' <p>')
old_question_hints = removetags(old_question_fields['hints'], 'p i img br').replace(' ','').strip()
old_question_content = json.loads(old_question_fields['content'])
old_question_soundclips = json.loads(old_question_fields['soundclips'])
old_question_tag = old_question_fields['tags']
_get_question_tag = []
#prepare question tags
for tag in old_question_tag:
_get_question_tag.append(get_question_tag(int(tag))['fields']['name'])
_question_title = ''
_question_content = {'options':[],'instruction':None,'tags':[]}
_question_content_widgets_element = question_widgets
_question_content_widgets = _question_content_widgets_element['widgets']
_question_content_widgets_images = _question_content_widgets['images']
_question_content_widgets_sounds = _question_content_widgets['sounds']
sound_clip_counter = 0
img_counter = 0
question_content_status = False
if 'img' in old_question_title:
_question_title_without_html = removetags(old_question_title, 'p br img strong').replace(' ','').strip()
all_imgs = get_image_from_content(old_question_title)
_question_title = ''
for img in all_imgs:
img_counter = img_counter + 1
img = download_media_from_path(img,'images')
_question_content_widgets_images.update({img_counter:img})
_question_title_without_html = _question_title_without_html+" [[img id=%s]]"%(img_counter)
_question_title = _question_title_without_html
else:
_question_title = old_question_title
# check sound in title and content
if 'content' in old_question_soundclips or 'question' in old_question_soundclips:
'''
1. prepare sound for title/question
2. prepare sound for content/options
'''
if 'question' in old_question_soundclips:
if old_question_soundclips['question']:
sound_clip_counter = sound_clip_counter+1
q_sound_path = old_question_soundclips['question']['path']
q_sound_path = download_media_from_path(q_sound_path,'sounds')
#audio to pic and question instruction logic
for tag in _get_question_tag:
sounds = "[[sound id=%d]]"%(sound_clip_counter)
if not tag.startswith('audioto'):
_question_content.update({'instruction':sounds})
_question_content_widgets_sounds.update({sound_clip_counter:q_sound_path})
_question_title = _question_title + removetags(old_question_title, 'p br img em strong').replace(' ','').strip()
if '[[img ' in _question_title:
_question_title = _question_title+" [[sound id=%d]] "%(sound_clip_counter)
else:
_question_title_without_html = removetags(old_question_title, 'p br img em strong').replace(' ','').strip()
_question_title = _question_title_without_html + " [[sound id=%d]] "%(sound_clip_counter)
if 'content' in old_question_soundclips:
if old_question_soundclips['content']:
for key, value in old_question_soundclips['content'].items():
# check if img is available content
question_placeholder = ''
for content_key, content_value in old_question_content.items():
if content_key == key and 'img' in content_value:
_question_content_without_html = removetags(content_value, 'img p')
all_imgs = get_image_from_content(content_value)
for img in all_imgs:
img = download_media_from_path(img,'images')
img_counter = img_counter + 1
_question_content_widgets_images.update({img_counter:img})
question_placeholder = _question_content_without_html+" [[img id=%s]] "%(img_counter)
# q_sound_path = old_question_soundclips['question']['path']
# q_sound_path = download_media_from_path(q_sound_path,'sounds')
sound_clip_counter = sound_clip_counter+1
q_sound_path = download_media_from_path(value,'sounds')
_question_content_widgets_sounds.update({sound_clip_counter:q_sound_path})
question_placeholder = question_placeholder+" [[sound id=%d]] "%(sound_clip_counter)
_question_content['options'].append({'key':mapper[key],'option':question_placeholder})
# _question_content.update(_question_content_widgets_sounds)
question_content_status = True
else:
pass
# logger.info('no question sound or no question image found.')
else:
pass
if question_content_status == False:
if isinstance(old_question_content,dict):
for key, value in old_question_content.items():
value_without_html = removetags(value, 'p br img').replace(' ','').strip()
all_imgs = get_image_from_content(value)
if not all_imgs:
_question_content['options'].append({'key':mapper[key],'option':value_without_html})
else:
for img in all_imgs:
img_counter = img_counter + 1
img = download_media_from_path(img,'images')
_question_content_widgets_images.update({img_counter:img})
value_without_html = value_without_html+" [[img id=%s]]"%(img_counter)
_question_content['options'].append({'key':mapper[key],'option':value_without_html})
elif isinstance(old_question_content,list):
_question_content = {'option':[]}
# values = removetags(key, 'p br img').replace(' ','').strip()
_question_content['option'].append(old_question_content)
elif old_question_content:
#values = removetags(old_question_content, 'p br img').replace(' ','').strip()
_question_content['option'].append(old_question_content)
_question_content.update(_question_content_widgets_element)
_question_content.update({'hints':old_question_hints})
_question_title = removetags(_question_title, 'p br img strong em').replace(' ','').strip()
if _question_title.startswith('.'):
_question_title = _question_title.replace('.',' ')
_question_content['tags']=_get_question_tag
return _question_title, _question_content
def set_msg(request, name, msg, audit=False):
"""
Método interno, que asigna el mensaje en la variable messages de django
y, deacuerdo al tipo de mensaje recibido en name, guarda en
logger.info() #por ejemplo.
Level Constant Value
DEBUG 10
INFO 20
SUCCESS 25
WARNING 30
ERROR 40
messages.debug(request, '%s SQL statements were executed.' % count) # no imprime
messages.info(request, 'Three credits remain in your account.')
messages.success(request, 'Profile details updated.')
messages.warning(request, 'Your account expires in three days.')
messages.error(request, 'Document deleted.')
"""
messagesToCall = getattr(messages, "info")
messagesToCall(request, (u''
u'<div class="alert alert-block alert-%s"><button type="button" class="close" data-dismiss="alert">×</button>%s</div>'
u'' % (name, force_text(msg)))
)
# m = __import__ ('logger')
if audit:
methodToCall = getattr(logger, name) # logger.debug donde name=debug p.e.
methodToCall(("[%s][%s][%s] %s ") % (request.get_full_path(), request.user, request.META['REMOTE_ADDR'], removetags(msg, 'b'))) # logger.debug(msg) #(str(u'%s'%msg)).encode('utf-8')
def set_msg(request, name, msg):
"""
Método interno, que deacuerdo al tipo de mensaje recibido en name, guarda en
logger.info() #por ejemplo.
"""
# m = __import__ ('logger')
methodToCall = getattr(logger, name) # logger.debug donde name=debug p.e.
methodToCall(("[%s][%s][%s] %s ") % (request.get_full_path(), request.user, request.META['REMOTE_ADDR'], removetags(msg, 'b'))) # logger.debug(msg) #(str(u'%s'%msg)).encode('utf-8')
def save(self,force_insert=False, force_update=False):
self.message = removetags(self.message,'script')
self.message = removetags(self.message,'iframe')
self.message = mark_safe(render_bbcode(self.message))
super(Message,self).save(force_insert=force_insert,force_update=force_update)
