def html_urlize(value, autoescape=None):
"""Converts URLs in html into clickable links."""
from BeautifulSoup import BeautifulSoup
ignored_tags = ['a', 'code', 'pre']
soup = BeautifulSoup(value)
tags = soup.findAll(True)
text_all = soup.contents
for text in text_all:
if text not in tags:
parsed_text = urlize(text, nofollow=True, autoescape=autoescape)
text.replaceWith(parsed_text)
for tag in tags:
if not tag.name in ignored_tags:
soup_text = BeautifulSoup(str(tag))
if len(soup_text.findAll()) > 1:
for child_tag in tag.contents:
child_tag.replaceWith(html_urlize(str(child_tag)))
elif len(soup_text.findAll()) > 0:
text_list = soup_text.findAll(text=True)
for text in text_list:
parsed_text = urlize(text, nofollow=True, autoescape=autoescape)
text.replaceWith(parsed_text)
try:
tag.replaceWith(str(soup_text))
except:
pass
return mark_safe(str(soup))
def as_html(self, tag='div'):
"""
Return an HTML representation of this chat, including tags marking
the author and text selection accordingly.
Use the tag argument to customize the tag that wraps each line in
a chat.
"""
html = u''
for line in self.text.splitlines():
line_sections = escape(line).split(': ', 1)
if len(line_sections) > 1:
html += u'<{tag} class="line"><span class="author">{author}: </span><span class="text">{text}</span><span class="post-text"></span></{tag}>'.format(
author=line_sections[0],
tag=tag,
text=urlize(line_sections[1]),
)
else:
html += u'<{tag} class="no-author line"><span class="text">{text}</span><span class="post-text"></span></{tag}>'.format(
tag=tag,
text=urlize(line_sections[0]),
)
return html
def get_context_data(self, **kwargs):
context = super(LoadDocs, self).get_context_data(**kwargs)
files = os.listdir(self.path_name)
context['files'] = dict([(file.split('.')[0].replace('_', ' ').title(), file) for file in files])
try:
context['file'] = urlize(open(os.path.join(settings.BASE_DIR, self.file_name)).read()).replace('\n', '<br>')
except:
print(self.file_name)
context['file'] = urlize(open(os.path.join(settings.BASE_DIR, self.path_name, context['files'][self.file_name])).read()).replace('\n', '<br>')
return context
def handle(self, *args, **options):
twitter_hashtags = TwitterHashtag.objects.all()
twitter_users = TwitterUser.objects.all()
n_tweets = 0
for user in twitter_users:
print user.name
user_timeline= twitter.get_user_timeline(screen_name=user.name,count=20)
for tweet in user_timeline:
guid = tweet['id']
ts = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
text = urlize(tweet['text'])
try:
tweet_obj = Tweet.objects.get(guid=guid)
except Tweet.DoesNotExist:
tweet_obj = Tweet.objects.create(
guid=guid,
text=text,
created_at=ts,
)
tweet_obj.save()
n_tweets += 1
for hashtag in twitter_hashtags:
print hashtag.name
result = twitter.search(q=hashtag.name,count=20)
tweets = result['statuses']
for tweet in tweets:
guid = tweet['id']
ts = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
text = urlize(tweet['text'])
try:
tweet_obj = Tweet.objects.get(guid=guid)
except Tweet.DoesNotExist:
tweet_obj = Tweet.objects.create(
guid=guid,
text=text,
created_at=ts,
)
tweet_obj.save()
n_tweets += 1
self.stdout.write('Successfully imported "%s" tweets ' % n_tweets)
def insert_post(self, id_user, first_name, last_name, username, avatar, postText, extra=None):
post = {
"id_user": id_user,
"first_name": first_name,
"last_name":last_name,
"username": "******" % (username),
"avatar": avatar,
"date": datetime.utcnow().isoformat(),
"text": urlize(escape(postText)),
"children": [],
"favourite": [],
"shared": 0,
}
post_id = super(Microblog,self).insert_post(post)
# xAPI
if extra and 'course_slug' in extra:
user = User.objects.get(pk=id_user)
try:
course = Course.objects.get(slug=extra['course_slug'])
except:
course = None
resource = {
'type': 'microblogpost',
'url': 'https://%s%s#%s' % (settings.API_URI, reverse('profile_posts_byid', kwargs={'id': id_user}), post_id),
'name': 'Microblog post',
'description': 'This is a blog post'
}
geolocation = extra.get('geolocation')
x_api.learnerSubmitsAResource(user, resource, course, geolocation)
def message_send(self, message_body, chat_uuid):
try:
logging.info('req_message_send from %s with message_body %s' % (self.user, message_body))
chat = Chat.objects.get(uuid=chat_uuid)
if not self.user in chat.users.all():
return
message = chat.add_message(self.user, urlize(escape(message_body)))
message_obj = prepare_for_emit(serializers.MessageSerializer(message).data)
user_chat_statuses = chat.user_chat_statuses
user_chat_statuses_obj = prepare_for_emit(
[CustomUserChatStatusSerializer(ucs).data for ucs in user_chat_statuses.all()])
for user in chat.users.all():
for connection in self.connections.get(user, []):
connection.emit('ev_message_sent', message_obj, user_chat_statuses_obj)
# Activate any archived user_chat_statusses
for user_chat_status in chat.user_chat_statuses.all().filter(status=UserChatStatus.ARCHIVED):
user_chat_status.activate()
chat_obj = prepare_for_emit(CustomChatSerializer(chat).data)
for connection in self.connections.get(user_chat_status.user, []):
connection.emit('ev_chat_created', chat_obj)
except Exception, e:
logging.error(str(e))
def render_content(content):
content = html.escape(content)
# Do the substitution
content = youtube_tag.sub(r'\g<0>\n<iframe width="560" height="315" src="http://www.youtube.com/embed/\1" frameborder="0" allowfullscreen></iframe>\n', content)
content = html.urlize(content)
content = a_tag.sub(r'<a href="\1" target="_blank">', content)
return content
def save(self, force_insert=False, force_update=False,
update_number_of_upcoming_gigs=False):
"""
Update the ``biography_html`` and ``number_of_upcoming_gigs``
fields.
Convert the plain-text ``biography`` field to HTML using Markdown
and store it in the ``biography_html`` field.
Update the number of upcoming gigs for this artist. The update is
bypassed by default, but can be forced by passing setting the
third argument, ``update_number_of_upcoming_gigs``, to ``True``.
The implication is that this should only be updated via the
``import_gigs_from_ripping_records`` command.
"""
if update_number_of_upcoming_gigs:
upcoming_gigs = Gig.objects.upcoming(
artist__id=self.id).values('artist_id')
gig_count = upcoming_gigs.annotate(num_of_artists=Count('id'))
try:
ordered_gig_count = gig_count.order_by('-num_of_artists')[0]
self.number_of_upcoming_gigs = ordered_gig_count['num_of_artists']
except IndexError:
self.number_of_upcoming_gigs = 0 # No gigs yet.
self.biography_html = markdown(urlize(self.biography, trim_url_limit=40,
nofollow=False))
super(Artist, self).save(force_insert, force_update)
def render_activity(item):
if not item.group:
# not implemented
return
action_str = ACTIVITY_ACTION_STRINGS[item.type]
if item.type == Activity.CREATE_ISSUE:
action_str = action_str.format(**item.data)
output = ''
if item.user:
user = item.user
name = user.first_name or user.email
output += '<span class="avatar"><img src="%s"></span> ' % (get_gravatar_url(user.email, size=20),)
output += '<strong>%s</strong> %s' % (escape(name), action_str)
else:
output += '<span class="avatar sentry"></span> '
output += 'The system %s' % (action_str,)
output += ' <span class="sep">—</span> <span class="time">%s</span>' % (timesince(item.datetime),)
if item.type == Activity.NOTE:
output += linebreaks(urlize(escape(item.data['text'])))
return mark_safe(output)
def test_urlize_with_label_templatetag(self):
bag1_files = self.bag1.list_payload()
bag1_url = invapp_extras.urlize_with_label(
urlize(self.bag1.access_url() + bag1_files[0][0]),
bag1_files[0][0])
self.assertEqual(bag1_url, '<a href="http://dspace.wrlc.org/tmp/'
'fakebag/data/METADATA/0123456789-dc.xml">'
'/data/METADATA/0123456789-dc.xml</a>')
bag2_files = self.bag2.list_payload()
bag2_url = invapp_extras.urlize_with_label(
urlize(self.bag2.access_url() + bag2_files[0][0]),
bag2_files[0][0])
self.assertEqual(bag2_url, '/data/METADATA/0123456789-dc.xml')
def get_rendered(self):
field_value = getattr(self, _name)
mt = getattr(self, "%s_markup_type" % name)
if mt == markup_settings.MARKUP_PLAIN_TEXT:
field_value = field_value.strip()
if field_value:
field_value = escape(field_value)
try:
# try to urlize if there are no invalid IPv6 URLs
field_value = urlize(field_value)
except ValueError:
pass
field_value = linebreaks(field_value)
elif mt == markup_settings.MARKUP_RAW_HTML:
pass
elif mt == markup_settings.MARKUP_HTML_WYSIWYG:
pass
elif mt == markup_settings.MARKUP_MARKDOWN:
try:
import markdown
except ImportError:
if settings.DEBUG:
raise Warning("Error in {% markdown %} filter: The Python markdown library isn't installed.")
return field_value
else:
field_value = markdown.markdown(field_value)
# remove empty paragraphs
field_value = field_value.replace('<p></p>', '')
return mark_safe(field_value)
def _output(self):
"""It is this method that is responsible for rendering the
object in HTML.
"""
if self.text is None or self.text == "":
return u""
if self.text_format == '\E':
return linebreaks(urlize(escape(self.text)))
elif self.text_format == '\T':
try:
return textile(self.text)
except:
return mark_safe(_('There is an error in the Textile.'))
elif self.text_format == '\M':
try:
return markdown(self.text, MARKDOWN_EXTENSIONS)
except:
return mark_safe(_('There is an error in the Markdown.'))
elif self.text_format == '\R':
try:
return restructuredtext(self.text)
except:
return mark_safe(_('There is an error in the reStructuredText.'))
elif self.text_format == '\H' or self.text_format == '\W':
return mark_safe(self.text)
else:
return mark_safe(escape(self.text))
def twitterize(value, autoescape=None):
# twitpic / twitvid/ yfrog support
append = u""
try:
v = value.lower()
if (
"http://twitpic.com/" in v
or "http://yfrog.com/" in v
or "http://twitvid.com/" in v
or "youtube.com/watch?" in v
):
for w in value.split():
wx = w.lower()
if wx.startswith("http://twitpic.com/"):
img_id = w.split("/")[-1]
append = append + u'<a href="%s"><img src="http://twitpic.com/show/mini/%s"/></a>' % (w, img_id)
elif wx.startswith("http://yfrog.com/") and w[-1] in _YFROG_SUFFIXES:
img_id = w
append = append + u'<a href="%s"><img src="%s.th.jpg"/></a>' % (w, img_id)
elif wx.startswith("http://twitvid.com/"):
img_id = w.split("/")[-1]
append = (
append
+ u"""<object width="425" height="344">
<param name="movie" value="http://www.twitvid.com/player/%s>"></param>
<param name="allowFullScreen" value="true"></param>
<embed type="application/x-shockwave-flash" src="http://www.twitvid.com/player/%s"
quality="high" allowscriptaccess="always" allowNetworking="all"
allowfullscreen="true" wmode="transparent" height="344" width="425"></embed>
</object>"""
% (img_id, img_id)
)
elif wx.startswith("http://youtube.com/") or wx.startswith("http://www.youtube.com/"):
# Example URL: http://www.youtube.com/watch?v=Apadq9iPNxA
pieces = urlparse.urlsplit(w)
img_id = cgi.parse_qs(pieces.query)["v"][0]
append = (
append
+ u"""<object width="425" height="344">
<param name='movie' value='http://www.youtube.com/v/%s&hl=en&fs=1'></param>
<param name='allowFullScreen' value='true'></param>
<embed src='http://www.youtube.com/v/%s&hl=en&fs=1'
type='application/x-shockwave-flash' allowfullscreen='true'
height="344" width="425"></embed>
</object>"""
% (img_id, img_id)
)
except:
pass
# Link URLs
value = urlize(value, nofollow=False, autoescape=autoescape)
# Link twitter usernames prefixed with @
value = re.sub(r"(\s+|\A)@([a-zA-Z0-9\-_]*)\b", r'\1<a href="http://twitter.com/\2">@\2</a>', value)
# Link hash tags
value = re.sub(
r"(\s+|\A)#([a-zA-Z0-9\-_]*)\b", r'\1<a href="http://search.twitter.com/search?q=%23\2">#\2</a>', value
)
if append:
value = value + u"<br/>" + append
return mark_safe(value)
def do_move(self, move, *args, **kw):
assert self.active
assert (self.gameplayer_set.get(user=self.current_player).player_num
== self.game_instance.player)
assert move in ('play_tiles', 'skip', 'swap')
g = self.game_instance
player_num = g.player
ret = getattr(g, move)(*args, **kw)
# Update fields to reflect change in game state
self.current_player = self.get_player(g.player)
self.last_played = datetime.now()
self.turn += 1
if g.winners:
for winner in g.winners:
gp = self.gameplayer_set.get(player_num=winner)
gp.winner = True
gp.save()
self.active = False
self.save()
if move == 'play_tiles':
words = ('%s: %d' % (word, score) for (word, _, _), score
in ret['words'].iteritems())
msg = 'scored %d points\n%s' % (ret['score'], '\n'.join(words))
elif move == 'skip':
msg = 'skipped'
else:
msg = 'swapped %d tiles' % ret
msg = urlize(msg, autoescape=True).replace('\n', '<br />')
pubsub.publish_move(self.id, player_num, msg)
return ret
def text2html(text):
#text = to_unicode(text).decode("utf-8", "replace")
text = unicode(text)
text = cgi.escape(text)
text = urlize(text, None)
text = text.replace(u'\n', u'\n<br />')
return text
def escape_wiki(text, lookup = False, autoescape=None):
"""
This safely escapes the HTML content and replace all links, @, etc by their TN counterpart.
We've got two versions:
- the fast one which doesn't lookup actual values
- the slow one which may wake every single object mentionned in the text.
Use whichever suits you the most.
"""
# standard text content (statusupdate, description, ...)
if not isinstance(text, SafeData):
# strip all tags
text = re.compile(r'<.*?>').sub('', text)
# clean escape
text = conditional_escape(text)
# urlize the links
text = urlize(text)
# replace linebreaks
text = text.replace('\n', '<br />')
# html rich content (using safe filter)
else:
text = conditional_escape(text)
# Replace the global matches
for regex, fast_reverse, func, model_class, lookup_field in matches:
subf = Subf(lookup, fast_reverse, func, model_class, lookup_field )
text = regex.sub(subf, text)
return mark_safe(text)
def render_comment(comment_obj, count, user, request):
#ok this is as ugly as it gets, but there's little other ways to generate this html that I am aware of
content_type = ContentType.objects.get_for_model(comment_obj.user)
path = "detail.html?_t=" + str(comment_obj.content_type.pk) + "&_o=" + str(comment_obj.object_pk)
form = ReplyForm(initial={'is_root': False, 'is_leaf': True, 'content_type': comment_obj.content_type,
'object_pk': comment_obj.object_pk, 'reply_to': comment_obj, 'submit_date': datetime.datetime.now(), 'user': user})
"""returns the relevant html for a single atomic comment given the comment object"""
try:
img = IMGSource.objects.get(object_pk=comment_obj.user.id, current=True)
ts = img.url + '=s20-c'
except:
ts = '/static/img/avatar_20x18.jpg'
html = "<a href='/user_profile.html" + "?_t=" + str(content_type.pk) + "&_o=" + str(comment_obj.user.pk) + "'" + ' class="avatar"><img src="' + ts + '" alt="' + str(user.username) + '"></a>'
html += "<li id='comment" + str(comment_obj.id) + "'>"
html += "<a href='/user_profile.html" + "?_t=" + str(content_type.pk) + "&_o=" + str(comment_obj.user.pk) + "'" + ' class="author">' + str(comment_obj.user.username) + '</a> <span class="meta">' + generate_time_string(comment_obj.submit_date, datetime.datetime.now()) + '</span><p>'
text = markdown(comment_obj.text, safe_mode=True)
html += smart_str(text, encoding='utf-8', strings_only=False, errors='strict') + '</p>'
if user.is_authenticated():
html += '<ul class="comment_links">' + '<li>' + "<a href='javascript:;' onmousedown=" + "'toggleSlide(" + '"add_reply' + str(comment_obj.id) + '"' + ");'>reply</a>" + '</li>' + '<li>' + "<a href='/" + path + + str(comment_obj.id) + "'>permalink</a>" + '</li>'
else:
html += '<ul class="comment_links">' + '<li>' + "<a href='/" + path + "&_i=s'>reply</a>" + '</li>' + '<li>' + "<a href='/" + path + "&_c=comment" + str(comment_obj.id) + "'>permalink</a>" + '</li>'
if comment_obj.user == user:
html += '<li>' + "<a href='javascript:;' onmousedown=" + "'toggleSlide(" + '"edit_reply' + str(comment_obj.id) + '"' + ");'>edit</a>" + '</li>'
html += '</ul>'
html += '<p>'
if user.is_authenticated():
html += "<div class='reply_comment' id='add_reply" + str(comment_obj.id) + "' style='display:none; overflow:hidden; height:290px;width:100%;'><form id='add_reply_form" + str(comment_obj.id) +"' method='post' action=''><div style='display:none'><input type='hidden' name='csrfmiddlewaretoken' value='" + str(request.COOKIES.get('csrftoken')) + "' /><input id='reply_to_object' type='hidden' name='reply_to_object' value='" + str(comment_obj.id)+ "'/></div>" + str(form.as_p()) + "<input type='submit' class='button' value='Submit'></form></div>"
if comment_obj.user == user:
editform = CommentForm(instance=comment_obj)
html += "<div class='reply_comment' id='edit_reply" + str(comment_obj.id) + "' style='display:none; overflow:hidden; height:290px;width:100%;'><form id='add_reply_form" + str(comment_obj.id) +"' method='post' action=''><div style='display:none'><input type='hidden' name='csrfmiddlewaretoken' value='" + str(request.COOKIES.get('csrftoken')) + "' /><input id='edit_object' type='hidden' name='edit_object' value='" + str(comment_obj.id)+ "'/>" + '<input type="hidden" name="form_id" value="pp_comment_form' + str(comment_obj.id) + '" id="id_form_id"/>' + "</div>" + str(editform.as_p()) + "<input type='submit' class='button' value='Submit'></form></div>"
html += '</p>'
html += "</li>"
return urlize(html, trim_url_limit=30, nofollow=True)
def status_creation_handler(sender, **kwargs):
status = kwargs.get('instance', None)
created = kwargs.get('created', False)
if not created or not isinstance(status, Status):
return
# convert status body to markdown and bleachify
bl = Bleach()
status.status = urlize(status.status)
status.status = bl.clean(markdown(status.status), tags=TAGS)
status.save()
# fire activity
activity = Activity(
actor=status.author,
verb='http://activitystrea.ms/schema/1.0/post',
status=status,
)
if status.project:
activity.target_project = status.project
activity.save()
# Send notifications.
if activity.target_project:
activity.target_project.send_update_notification(activity)
def test_urlize_unchanged_inputs(self):
tests = (
('a' + '@a' * 50000) + 'a', # simple_email_re catastrophic test
('a' + '.' * 1000000) + 'a', # trailing_punctuation catastrophic test
)
for value in tests:
self.assertEqual(html.urlize(value), value)
def make_html(self, text):
"""Применяет базовую html-разметку к указанному тексту.
:param str text:
:rtype: str
"""
return urlize(text.replace('\n', '<br />'), nofollow=True)
def whiteboard_messages(project=None, component=None, language=None):
"""Display whiteboard messages for given context"""
ret = []
whiteboards = WhiteboardMessage.objects.context_filter(
project, component, language
)
for whiteboard in whiteboards:
if whiteboard.message_html:
content = mark_safe(whiteboard.message)
else:
content = mark_safe(urlize(whiteboard.message, autoescape=True))
ret.append(
render_to_string(
'message.html',
{
'tags': ' '.join((whiteboard.category, 'whiteboard')),
'message': content,
}
)
)
return mark_safe('\n'.join(ret))
def urlizetrunc(value, limit):
"""
Converts URLs into clickable links, truncating URLs to the given character limit
Argument: Length to truncate URLs to.
"""
from django.utils.html import urlize
return urlize(value, trim_url_limit=int(limit), nofollow=True)
def urlizenf(text):
"""
Same as urlize except it transforms the links to have rel="nofollow"
>>> www.someurl.com
<a href="http://www.someurl.com" rel="nofollow">www.someurl.com</a>
"""
from django.utils.html import urlize
return urlize(text, None, True, True)
def getNDescription(self):
if self.description is not None :
ntext = self.description.strip()
ntext = strip_tags(ntext)
ntext = ntext.strip()
ntext = escape(ntext)
ntext = urlize(ntext)
return ntext.replace("\n","<br/>")
def to_native(self, value):
# Convert model instance text -> text for reading.
text = super(ContentTextField, self).to_native(value)
# This is equivalent to the django template filter: '{{ value|urlize|linebreaks }}'. Note: Text from the
# database is escaped again here (on read) just as a double check for HTML / JS injection.
text = linebreaks(urlize(text, None, True, True))
# This ensure links open in a new window (BB-136).
return re.sub(r'<a ', '<a target="_blank" ', text)
def convertToProperHTML(text):
ntext = text.strip()
ntext = strip_tags(ntext)
ntext = ntext.strip()
ntext = escape(ntext)
ntext = urlize(ntext)
return ntext.replace("\n","<br/>")
def getNText(self):
ntext = self.text.strip()
ntext = strip_tags(ntext)
ntext = ntext.strip()
ntext = escape(ntext)
ntext = urlize(ntext)
return ntext.replace("\n","<br/>")
def run(self):
"""
Request new tweets from the Twitter API.
"""
urls = {
QUERY_TYPE_USER: ("http://api.twitter.com/1/statuses/"
"user_timeline/%s.json?include_rts=true" %
self.value.lstrip("@")),
QUERY_TYPE_LIST: ("http://api.twitter.com/1/%s/statuses.json"
"?include_rts=true" %
self.value.lstrip("@").replace("/", "/lists/")),
QUERY_TYPE_SEARCH: "http://search.twitter.com/search.json?q=%s" %
quote(self.value.encode("utf-8")),
}
try:
url = urls[self.type]
except KeyError:
return
try:
tweets = loads(urlopen(url).read())
except:
return
if self.type == "search":
tweets = tweets["results"]
for tweet_json in tweets:
remote_id = str(tweet_json["id"])
tweet, created = self.tweets.get_or_create(remote_id=remote_id)
if not created:
continue
if "retweeted_status" in tweet_json:
user = tweet_json["user"]
tweet.retweeter_user_name = user["screen_name"]
tweet.retweeter_full_name = user["name"]
tweet.retweeter_profile_image_url = user["profile_image_url"]
tweet_json = tweet_json["retweeted_status"]
if self.type == QUERY_TYPE_SEARCH:
tweet.user_name = tweet_json["from_user"]
tweet.full_name = tweet_json["from_user"]
tweet.profile_image_url = tweet_json["profile_image_url"]
date_format = "%a, %d %b %Y %H:%M:%S +0000"
else:
user = tweet_json["user"]
tweet.user_name = user["screen_name"]
tweet.full_name = user["name"]
tweet.profile_image_url = user["profile_image_url"]
date_format = "%a %b %d %H:%M:%S +0000 %Y"
tweet.text = urlize(tweet_json["text"])
tweet.text = re_usernames.sub(replace_usernames, tweet.text)
tweet.text = re_hashtags.sub(replace_hashtags, tweet.text)
if getattr(settings, 'TWITTER_STRIP_HIGH_MULTIBYTE', False):
chars = [ch for ch in tweet.text if ord(ch) < 0x800]
tweet.text = ''.join(chars)
d = datetime.strptime(tweet_json["created_at"], date_format)
d -= timedelta(seconds=timezone)
tweet.created_at = make_aware(d)
tweet.save()
self.interested = False
self.save()
def test_urlize(self):
self.assertEqual(
html.urlize('Hello http://google.com/?sdf=443&s="sdf" hello'),
'Hello <a href="http://google.com/?sdf=443&s=%22sdf">http://google.com/?sdf=443&s="sdf</a>" hello'
)
self.assertEqual(
html.urlize('Email [email protected]'),
'Email <a href="mailto:[email protected]">[email protected]</a>'
)
self.assertEqual(
html.urlize('Double urls http://example.com/?x=ä&ä=4 http://example.com/?x=ä&ä=2 test'),
'Double urls <a href="http://example.com/?x=%C3%A4&%C3%A4=4">http://example.com/?x=ä&ä=4</a> '
'<a href="http://example.com/?x=%C3%A4&%C3%A4=2">http://example.com/?x=ä&ä=2</a> test'
)
self.assertEqual(
html.urlize('Invalid url http://ex[ls.com/ does not get a tag'),
'Invalid url http://ex[ls.com/ does not get a tag'
)
self.assertEqual(
html.urlize('text <> and url http://example.com/<tag>?<q>=<v> gets escaped', autoescape=True),
'text <> and url <a href="http://example.com/%3Ctag%3E?%3Cq%3E=%3Cv%3E">'
'http://example.com/<tag>?<q>=<v></a> gets escaped'
)
self.assertEqual(
html.urlize('No follow http://example.com', nofollow=True),
'No follow <a href="http://example.com" rel="nofollow">http://example.com</a>'
)
self.assertEqual(
html.urlize('Truncate http://example.com', trim_url_limit=5),
'Truncate <a href="http://example.com">ht...</a>'
)
def urlizetrunc(value, limit):
"""
Converts URLs into clickable links, truncating URLs to the given character limit,
and adding 'rel=nofollow' attribute to discourage spamming.
Argument: Length to truncate URLs to.
"""
from django.utils.html import urlize
return urlize(value, trim_url_limit=int(limit), nofollow=True)
# Verbose names for the teams
VERBOSE_TEAMS = {
'H': "Humans",
'Z': "Zombies",
'B': "Both Teams",
}
# The length of a feed code.
FEED_LEN = 5
# The characters we allow in a feed code.
VALID_CHARS = ["A", "C", "E", "L", "X", "N", "P", "O", "S", "T", "W", "Z"]
MARKUP_FIELD_TYPES = (
('markdown', lambda markup: markdown.markdown(html.urlize(markup))),
('plain', lambda markup: html.urlize(html.linebreaks(markup))),
)
# The times corresponding to the beginning of "day" and "night". Used
# for missions.
START_TIMES = {
'D': datetime.time(hour=7),
'N': datetime.time(hour=17),
}
WSGI_APPLICATION = 'passenger_wsgi.application'
MOD_PHONE_NUMBER = "909-555-5555"
MODERATOR_EMAIL = "*****@*****.**"
def input_data_details(page, request, id):
data = InputData.objects.get(id=id)
template = 'pages/input_data_details_page.html'
if data.description:
description = urlize(linebreaks(data.description))
else:
description = page.input_data_description or ''
if request.user.is_superuser:
description += ' <a href="{}">admin edit</a>'.format(
urlresolvers.reverse('admin:climatemodels_inputdata_change',
args=(data.id, )))
subpage = {'title': 'Input data set: %s' % data.name, 'url': ''}
context = {
'page':
page,
'subpage':
subpage,
'description':
description,
'list': [
{
'notoggle':
True,
'opened':
True,
'definitions': [
{
'text': 'Data Type: %s' % data.data_type
},
{
'text':
'Simulation rounds: %s' % ', '.join(
(x.name for x in data.simulation_round.all()))
},
{
'text':
'Scenarios: %s' % ', '.join(
(x.name for x in data.scenario.all()))
},
{
'text':
'Variables: %s' % ', '.join(
(x.as_span() for x in data.variables.all()))
},
]
},
{
'notoggle': True,
'opened': True,
'term': 'Specifications',
'definitions': [{
'text': data.specification
}]
},
{
'notoggle': True,
'opened': True,
'term': 'Data source',
'definitions': [{
'text': data.data_source
}]
},
{
'notoggle': True,
'opened': True,
'term': 'Caveats',
'definitions': [{
'text': urlize(linebreaks(data.caveats))
}]
},
{
'notoggle':
True,
'opened':
True,
'term':
'Download Instructions',
'definitions': [{
'text':
urlize(linebreaks(data.download_instructions))
}]
},
]
}
return render(request, template, context)
})
def smile_it(str):
s = str
for smile, url in PYBB_SMILES.items():
s = s.replace(
smile, '<img src="%s%s%s" alt="smile" />' %
(settings.STATIC_URL, PYBB_SMILES_PREFIX, url))
return s
PYBB_MARKUP_ENGINES = getattr(
settings, 'PYBB_MARKUP_ENGINES', {
'bbcode':
lambda str: urlize(
smile_it(render_bbcode(str, exclude_tags=['size', 'center']))),
'markdown':
lambda str: urlize(smile_it(Markdown(safe_mode='escape').convert(str)))
})
PYBB_QUOTE_ENGINES = getattr(
settings, 'PYBB_QUOTE_ENGINES', {
'bbcode':
lambda text, username="": '[quote="%s"]%s[/quote]\n' %
(username, text),
'markdown':
lambda text, username="": '>' + text.replace('\n', '\n>').replace(
'\r', '\n>') + '\n'
})
PYBB_MARKUP = getattr(settings, 'PYBB_MARKUP', 'bbcode')
def _urlize_dict_check(self, data):
"""
For all items in dict test assert that the value is urlized key
"""
for original, urlized in data.items():
assert urlize(original, nofollow=False) == urlized
def markup2html(comment):
new_comment = ''
# For code blocks
code = False
prev_line = True
lines = comment.split('\n')
for index, line in enumerate(lines):
if re.match(r'^$', line):
if code:
if not prev_line:
new_comment += '\n'
else:
new_comment += '<p>'
prev_line = line
continue
# Making sure * won't be part of urls
html.TRAILING_PUNCTUATION += [u'.*', u'*']
# Create urls
try:
line = html.urlize(line, 63, True)
except ValueError:
# Temporarily fix for https://code.djangoproject.com/ticket/19070
pass
prev_line = line
new_line = line
# Starting with double space means it's a code block
code_block = re.search(r'^ ', new_line)
if code_block:
if not code:
new_line = '<p><pre><code>' + new_line
code = True
else:
new_line = '\n' + new_line
else:
# Replacing * with <i> or </i>
start = True
# Index offset
j = 0
asterisks = []
# First getting all asterisk that should be replaced
# * not followed by " rel= or </a> (basically * in links)
for x in re.finditer(r'\*((?!</a>)(?!" rel=))', line):
i = x.start(0)
try:
prev_char = line[i - 1:i]
except IndexError:
prev_char = None
try:
next_char = line[i + 1:i + 2]
except IndexError:
next_char = None
prev_ws = re.match(r'\s', prev_char)
next_ws = re.match(r'\s', next_char)
# Replace all * with italic tag if it is either preceded or followed by another character
# This means that ** is valid, but * * is not
# The three steps check for the following:
# "^*\S" or " *\S"
# "\S*\S"
# # "\S* " or "\S*$"
# Which creates this in code:
# if ((not prev_char or prev_ws) and not next_ws) or \
# (prev_char and not prev_ws and (next_char and not next_ws)) or \
# (prev_char and not prev_ws and (not next_char or next_ws)):
# Shortened version of previous code
if not next_ws or (prev_char and not prev_ws):
asterisks.append(i)
# Replace all found asterisks
if len(asterisks) > 1:
for i, asterisk in enumerate(asterisks):
# If there is an odd number of asterisks leave the last one
if (i + 1) == len(asterisks) and len(asterisks) % 2:
continue
if start:
italic = '<i>'
else:
italic = '</i>'
start = not start
# Replacing in line with some offset corrections
new_line = new_line[0:asterisk +
j] + italic + new_line[asterisk + j +
1:]
# Adding offset (string has more letters after adding <i>)
j += len(italic) - 1
if not code_block:
if code:
# Ending code tag
new_line = '</code></pre></p>' + new_line
code = False
else:
new_line += ' '
# Append proper closing tags if line is last and in a code block
if code and index == (len(lines) - 1):
new_line += '</code></pre></p>'
new_comment += new_line
return new_comment
def sanitize(value, extra_filters=None):
"""
Sanitize the given HTML.
Based on code from:
* http://www.djangosnippets.org/snippets/1655/
* http://www.djangosnippets.org/snippets/205/
"""
if value is None:
return u''
if '<' not in value and '&#' not in value and \
re.search(r'&\w+;', value) is None: # no HTML
# convert plain-text links into HTML
return mark_safe(
urlize(value, nofollow=True,
autoescape=True).replace('\n', '<br/>'))
js_regex = re.compile(r'[\s]*(&#x.{1,7})?'.join(list('javascript')),
re.IGNORECASE)
allowed_tags = ('p i strong em b u a h1 h2 h3 h4 h5 h6 pre br img ul '
'ol li span').split()
allowed_attributes = 'href src style'.split()
whitelist = False
extra_tags = ()
extra_attributes = ()
if isinstance(extra_filters, basestring):
if '|' in extra_filters:
parts = extra_filters.split('|')
else:
parts = [extra_filters.split()]
if parts[0] == 'whitelist':
whitelist = True
parts = parts[1:]
extra_tags = parts[0].split()
if len(parts) > 1:
extra_attributes = parts[1].split()
elif extra_filters:
extra_tags = extra_filters
if whitelist:
allowed_tags, allowed_attributes = extra_tags, extra_attributes
else:
allowed_tags = set(allowed_tags) - set(extra_tags)
allowed_attributes = set(allowed_attributes) - set(extra_attributes)
soup = BeautifulSoup(value)
for comment in soup.findAll(text=lambda text: isinstance(text, Comment)):
# remove comments
comment.extract()
for tag in soup.findAll(True):
if tag.name not in allowed_tags:
tag.hidden = True
else:
tag.attrs = [(attr, js_regex.sub('', val))
for attr, val in tag.attrs
if attr in allowed_attributes]
return mark_safe(soup.renderContents().decode('utf8'))
def plain_text_input_converter(text):
"""plain text to html converter"""
return sanitize_html(urlize('<p>' + text + '</p>'))
def parse(self, text):
return conditional_escape(mark_safe(linebreaks(urlize(escape(text)))))
def parse_post_text(post):
"""typically post has a field to store raw source text
in comment it is called .comment, in Question and Answer it is
called .text
also there is another field called .html (consistent across models)
so the goal of this function is to render raw text into .html
and extract any metadata given stored in source (currently
this metadata is limited by twitter style @mentions
but there may be more in the future
function returns a dictionary with the following keys
html
newly_mentioned_users - list of <User> objects
removed_mentions - list of mention <Activity> objects - for removed ones
"""
text = post.get_text()
if post._escape_html:
text = cgi.escape(text)
if post._urlize:
text = html.urlize(text)
if post._use_markdown:
text = sanitize_html(markup.get_parser().convert(text))
#todo, add markdown parser call conditional on
#post.use_markdown flag
post_html = text
mentioned_authors = list()
removed_mentions = list()
if '@' in text:
op = post.get_origin_post()
anticipated_authors = op.get_author_list(
include_comments = True,
recursive = True
)
extra_name_seeds = markup.extract_mentioned_name_seeds(text)
extra_authors = set()
for name_seed in extra_name_seeds:
extra_authors.update(User.objects.filter(
username__istartswith = name_seed
)
)
#it is important to preserve order here so that authors of post
#get mentioned first
anticipated_authors += list(extra_authors)
mentioned_authors, post_html = markup.mentionize_text(
text,
anticipated_authors
)
#find mentions that were removed and identify any previously
#entered mentions so that we can send alerts on only new ones
from askbot.models.user import Activity
if post.pk is not None:
#only look for previous mentions if post was already saved before
prev_mention_qs = Activity.objects.get_mentions(
mentioned_in = post
)
new_set = set(mentioned_authors)
for prev_mention in prev_mention_qs:
user = prev_mention.get_mentioned_user()
if user is None:
continue
if user in new_set:
#don't report mention twice
new_set.remove(user)
else:
removed_mentions.append(prev_mention)
mentioned_authors = list(new_set)
data = {
'html': post_html,
'newly_mentioned_users': mentioned_authors,
'removed_mentions': removed_mentions,
}
return data
def urlize_newtab(value, autoescape=True):
"""Converts URLs in plain text into clickable links that open in new tabs."""
url = urlize(value, nofollow=True, autoescape=autoescape)
url = url.replace("a href", 'a target="_blank" rel="noreferrer" href')
return mark_safe(url)
def render(self):
if self.message_html:
return mark_safe(self.message)
return mark_safe(urlize(self.message, autoescape=True))
'PASSWORD': '',
}
}
else:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': 'markuptest.db'
}
}
def render_rest(markup):
parts = publish_parts(source=markup, writer_name="html4css1")
return parts["fragment"]
MARKUP_FIELD_TYPES = [
('markdown', markdown.markdown),
('ReST', render_rest),
('plain', lambda markup: urlize(linebreaks(escape(markup)))),
]
INSTALLED_APPS = ('markupfield.tests', )
SECRET_KEY = 'sekrit'
MIDDLEWARE_CLASSES = ()
ROOT_URLCONF = ()
else:
origin_author_html = ''
return '<blockquote>%s%s</blockquote>' % (origin_author_html, value)
bbcode_parser.add_formatter('quote',
_render_quote,
strip=True,
swallow_trailing_newline=True)
PYBB_MARKUP_ENGINES = getattr(
settings, 'PYBB_MARKUP_ENGINES', {
'bbcode':
lambda str: smile_it(bbcode_parser.format(str)),
'markdown':
lambda str: urlize(smile_it(Markdown(safe_mode='escape').convert(str)))
})
PYBB_QUOTE_ENGINES = getattr(
settings, 'PYBB_QUOTE_ENGINES', {
'bbcode':
lambda text, username="": '[quote="%s"]%s[/quote]\n' %
(username, text),
'markdown':
lambda text, username="": '>' + text.replace('\n', '\n>').replace(
'\r', '\n>') + '\n'
})
PYBB_MARKUP = getattr(settings, 'PYBB_MARKUP', 'bbcode')
PYBB_TEMPLATE = getattr(settings, 'PYBB_TEMPLATE', "base.html")
def tinymce_input_converter(text):
"""tinymce input to production html converter"""
text = urlize(text)
return strip_tags(text, ['script', 'style', 'link'])
def get_context_data(self, **kwargs):
context = super(LoadView, self).get_context_data(**kwargs)
context['file'] = urlize(
open(os.path.join(settings.BASE_DIR,
self.file_name)).read()).replace('\n', '<br>')
return context
def markdown(text):
text = markdown2.markdown(urlize(text, nofollow=True, autoescape=True))
html = bleach.clean(text, tags=settings.MARKDOWN_FILTER_WHITELIST_TAGS)
return mark_safe(bleach.linkify(html))
def markdown_input_converter(text):
"""markdown to html converter"""
text = urlize(text)
text = get_parser().convert(text)
return sanitize_html(text)
def urlize(value, autoescape=None):
"""Converts URLs in plain text into clickable links."""
from django.utils.html import urlize
return mark_safe(urlize(value, nofollow=True, autoescape=autoescape))
def chat(self, player, msg):
player_num = self.gameplayer_set.get(user=player).player_num
msg = urlize(msg, autoescape=True).replace('\n', '<br />')
pubsub.publish_chat(self.id, player_num, msg)
def run(self):
"""
Request new tweets from the Twitter API.
"""
try:
value = quote(self.value)
except KeyError:
value = self.value
urls = {
QUERY_TYPE_USER: ("https://api.twitter.com/1.1/statuses/"
"user_timeline.json?screen_name=%s"
"&include_rts=true" % value.lstrip("@")),
QUERY_TYPE_LIST: ("https://api.twitter.com/1.1/lists/statuses.json"
"?list_id=%s&include_rts=true" % value),
QUERY_TYPE_SEARCH: "https://api.twitter.com/1.1/search/tweets.json"
"?q=%s" % value,
}
try:
url = urls[self.type]
except KeyError:
raise TwitterQueryException("Invalid query type: %s" % self.type)
auth_settings = get_auth_settings()
if not auth_settings:
from mezzanine.conf import registry
if self.value == registry["TWITTER_DEFAULT_QUERY"]["default"]:
# These are some read-only keys and secrets we use
# for the default query (eg nothing has been configured)
auth_settings = (
"KxZTRD3OBft4PP0iQW0aNQ",
"sXpQRSDUVJ2AVPZTfh6MrJjHfOGcdK4wRb1WTGQ",
"1368725588-ldWCsd54AJpG2xcB5nyTHyCeIC3RJcNVUAkB1OI",
"r9u7qS18t8ad4Hu9XVqmCGxlIpzoCN3e1vx6LOSVgyw3R",
)
else:
raise TwitterQueryException("Twitter OAuth settings missing")
try:
tweets = requests.get(url, auth=OAuth1(*auth_settings)).json()
except Exception as e:
raise TwitterQueryException("Error retrieving: %s" % e)
try:
raise TwitterQueryException(tweets["errors"][0]["message"])
except (IndexError, KeyError, TypeError):
pass
if self.type == "search":
tweets = tweets["statuses"]
for tweet_json in tweets:
remote_id = str(tweet_json["id"])
tweet, created = self.tweets.get_or_create(remote_id=remote_id)
if not created:
continue
if "retweeted_status" in tweet_json:
user = tweet_json['user']
tweet.retweeter_user_name = user["screen_name"]
tweet.retweeter_full_name = user["name"]
tweet.retweeter_profile_image_url = user["profile_image_url"]
tweet_json = tweet_json["retweeted_status"]
if self.type == QUERY_TYPE_SEARCH:
tweet.user_name = tweet_json['user']['screen_name']
tweet.full_name = tweet_json['user']['name']
tweet.profile_image_url = \
tweet_json['user']["profile_image_url"]
date_format = "%a %b %d %H:%M:%S +0000 %Y"
else:
user = tweet_json["user"]
tweet.user_name = user["screen_name"]
tweet.full_name = user["name"]
tweet.profile_image_url = user["profile_image_url"]
date_format = "%a %b %d %H:%M:%S +0000 %Y"
tweet.text = urlize(tweet_json["text"])
tweet.text = re_usernames.sub(replace_usernames, tweet.text)
tweet.text = re_hashtags.sub(replace_hashtags, tweet.text)
if getattr(settings, 'TWITTER_STRIP_HIGH_MULTIBYTE', False):
chars = [ch for ch in tweet.text if ord(ch) < 0x800]
tweet.text = ''.join(chars)
d = datetime.strptime(tweet_json["created_at"], date_format)
tweet.created_at = make_aware(d, utc)
try:
tweet.save()
except Warning:
pass
tweet.save()
self.interested = False
self.save()
def default_text(text):
return mark_safe(linebreaks(urlize(escape(text))))
input_file = open(input_filename, "r") # open and parse
input_file_lines = input_file.readlines()
edited_text = ""
url_regex = '.*http[s]?://.*' # any url
url_md_regex = '\[.*\]\(http[s]?://.*\)' # hyperlink in mardown link ()[http...]
url2_md_regex = '\<http[s]?://.*\>' # another hyperlink in markdown <http...>
# add <url> scheme
# dont use url_regex
# do so, term by term
for line in input_file_lines:
line = line.decode("utf-8")
# search url # if is not already a mardown link
# bool(re.search(url_regex, line)) == True and
if bool(re.search(url_md_regex, line)) == False and bool(
re.search(url2_md_regex, line)) == False:
line_urlized = urlize(line, nofollow=False)
edited_text = edited_text + line_urlized
else:
edited_text = edited_text + line
print edited_text.encode('utf-8')
# edited_file = open(input_filename, 'w') #write
# edited_file.write(edited_text.encode("utf-8"))
# edited_file.close()
def is_url(url):
return urlize(url) != url
def urlize_follow(value):
return mark_safe(urlize(value, nofollow=False))
def impact_model_details(page, request, id):
try:
base_model = BaseImpactModel.objects.get(id=id)
except:
messages.warning(request, 'Unknown model')
return HttpResponseRedirect('/impactmodels/')
title = 'Impact model: %s' % base_model.name
subpage = {'title': title, 'url': ''}
context = {'page': page, 'subpage': subpage, 'headline': ''}
can_edit_model = False
if request.user.is_authenticated() and (
base_model in request.user.userprofile.owner.all()
or request.user.is_superuser):
can_edit_model = True
# context['editlink'] += ' | <a href="{}">admin edit</a>'.format(
# urlresolvers.reverse('admin:climatemodels_impactmodel_change', args=(impactmodel.id,)))
model_simulation_rounds = []
for im in base_model.impact_model.filter(public=True):
im_values = im.values_to_tuples() + im.fk_sector.values_to_tuples()
model_details = []
for k, v in im_values:
if any((y for x, y in v)):
res = {
'term':
k,
'definitions': ({
'text': "%s: <i>%s</i>" % (x, y),
'key': x,
'value': y
} for x, y in v if y)
}
model_details.append(res)
if model_details:
model_details[0]['opened'] = True
edit_link = ''
if can_edit_model:
edit_link = '<i class="fa fa-cog" aria-hidden="true"></i> <a href="{}">Edit model information for simulation round {}</a>'.format(
page.url + page.reverse_subpage(STEP_BASE, args=(im.id, )),
im.simulation_round.name)
model_simulation_rounds.append({
'simulation_round':
im.simulation_round.name,
'simulation_round_slug':
im.simulation_round.slug,
'model_name':
base_model.name,
'edit_link':
edit_link,
'details':
model_details
})
context['description'] = urlize(base_model.short_description or '')
context['model_simulation_rounds'] = model_simulation_rounds
context['model_name'] = base_model.name
bm_values = base_model.values_to_tuples()
for k, v in bm_values:
if any((y for x, y in v)):
res = {
'term':
k,
'definitions': ({
'text': "%s: <i>%s</i>" % (x, y),
'key': x,
'value': y
} for x, y in v if y),
'opened':
True
}
context['base_model'] = [
res,
]
template = 'climatemodels/details.html'
return render(request, template, context)
def render_plain(content_data):
return linebreaks(urlize(escape(content_data)))
def format_plaintext_for_html(string):
return html.linebreaks(html.urlize(html.escape(string)))
def urlize_with_target_blank(s):
return mark_safe(
urlize(s, nofollow=True,
autoescape=True).replace("<a ", '<a target="_blank" '))
def get_text_html(self, obj):
return urlize(obj.text)
def send_email(self, user, place):
subject = _("[Pasporta Servo] You received an Authorization")
to = [user.email]
email_template_text = 'hosting/emails/new_authorization.txt'
email_template_html = 'hosting/emails/mail_template.html'
email_context = {
'user_first_name': user.profile.name,
'owner_name': place.owner.full_name,
'place_id': place.pk,
'place_address': str(place),
'site_domain': self.request.get_host(),
'site_name': settings.SITE_NAME,
}
message_text = render_to_string(email_template_text, email_context)
message_html = render_to_string(email_template_html, {'body': mark_safe(tohtmlpara(urlize(message_text))),})
message = EmailMultiAlternatives(subject, message_text, to=to)
message.attach_alternative(message_html, 'text/html')
message.send()