def test():
"""Take some example URLs and print out a nice pretty HTML table
of their extracted thubmnails and media objects"""
import sys
from r2.lib.filters import websafe
print "<html><body><table border=\"1\">"
for url in test_urls:
sys.stderr.write("%s\n" % url)
print "<tr>"
h = make_scraper(url)
print "<td>"
print "<b>", websafe(url), "</b>"
print "<br />"
print websafe(repr(h))
img = h.largest_image_url()
if img:
print "<td><img src=\"%s\" /></td>" % img
else:
print "<td>(no image)</td>"
mo = h.media_object()
print "<td>"
if mo:
s = scrapers[mo['type']]
print websafe(repr(mo))
print "<br />"
print s.media_embed(**mo).content
else:
print "None"
print "</td>"
print "</tr>"
print "</table></body></html>"
def rendercontent(self, input, fp):
soup = BeautifulSoup(input)
output = soup.find("div", {'class': 'wiki', 'id': 'content'})
# Replace all links to "/wiki/help/..." with "/help/..."
for link in output.findAll('a'):
if link.has_key('href') and link['href'].startswith("/wiki/help"):
link['href'] = link['href'][5:]
# Add "edit this page" link if the user is allowed to edit the wiki
if c.user_is_loggedin and c.user.can_wiki():
edit_text = _('edit this page')
yes_you_can = _("yes, it's okay!")
read_first = _('just read this first.')
url = "http://code.reddit.com/wiki" + websafe(fp) + "?action=edit"
edittag = """
<div class="editlink">
<hr/>
<a href="%s">%s</a> 
(<b>%s 
<a href="/help/editing_help">%s</a></b>)
</div>
""" % (url, edit_text, yes_you_can, read_first)
output.append(edittag)
output = SC_OFF + unicode(output) + SC_ON
return HelpPage(_("help"),
content=Embed(content=output),
show_sidebar=None).render()
def rendercontent(self, input, fp):
soup = BeautifulSoup(input)
output = soup.find("div", { 'class':'wiki', 'id':'content'} )
# Replace all links to "/wiki/help/..." with "/help/..."
for link in output.findAll('a'):
if link.has_key('href') and link['href'].startswith("/wiki/help"):
link['href'] = link['href'][5:]
# Add "edit this page" link if the user is allowed to edit the wiki
if c.user_is_loggedin and c.user.can_wiki():
edit_text = _('edit this page')
yes_you_can = _("yes, it's okay!")
read_first = _('just read this first.')
url = "http://code.sciteit.com/wiki" + websafe(fp) + "?action=edit"
edittag = """
<div class="editlink">
<hr/>
<a href="%s">%s</a> 
(<b>%s 
<a href="/help/editing_help">%s</a></b>)
</div>
""" % (url, edit_text, yes_you_can, read_first)
output.append(edittag)
output = SC_OFF + unicode(output) + SC_ON
return HelpPage(_("help"),
content = Embed(content=output),
show_sidebar = None).render()
def _oembed_comment(thing, **embed_options):
link = thing.link_slow
subreddit = link.subreddit_slow
if (not can_view_link_comments(link)
or subreddit.type in Subreddit.private_types):
raise ForbiddenError(errors.COMMENT_NOT_ACCESSIBLE)
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
title = _('%(author)s\'s comment from discussion "%(title)s"') % {
"author": author_name,
"title": _force_unicode(link.title),
}
else:
author_name = ""
title = ""
parent = "true" if embed_options.get('parent') else "false"
html = format_html(
embeds.get_inject_template(embed_options.get('omitscript')),
media=g.media_domain,
parent=parent,
live="true" if embed_options.get('live') else "false",
created=datetime.now(g.tz).isoformat(),
comment=thing.make_permalink_slow(force_domain=True),
link=link.make_permalink_slow(force_domain=True),
title=websafe(title),
uuid=uuid1(),
)
oembed_response = dict(
_OEMBED_BASE,
type="rich",
title=title,
author_name=author_name,
html=html,
)
if author_name:
oembed_response['author_url'] = make_url_https('/user/' + author_name)
return oembed_response
def _oembed_post(thing, **embed_options):
subreddit = thing.subreddit_slow
if (not can_view_link_comments(thing)
or subreddit.type in Subreddit.private_types):
raise ForbiddenError(errors.POST_NOT_ACCESSIBLE)
live = ''
if embed_options.get('live'):
time = datetime.now(g.tz).isoformat()
live = 'data-card-created="{}"'.format(time)
script = ''
if not embed_options.get('omitscript', False):
script = format_html(
SCRIPT_TEMPLATE,
embedly_script=EMBEDLY_SCRIPT,
)
link_url = UrlParser(thing.make_permalink_slow(force_domain=True))
link_url.update_query(ref='share', ref_source='embed')
author_name = ""
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
html = format_html(
POST_EMBED_TEMPLATE,
live_data_attr=live,
link_url=link_url.unparse(),
title=websafe(thing.title),
subreddit_url=make_url_https(subreddit.path),
subreddit_name=subreddit.name,
script=script,
)
oembed_response = dict(
_OEMBED_BASE,
type="rich",
title=thing.title,
author_name=author_name,
html=html,
)
return oembed_response
def _oembed_comment(thing, **embed_options):
link = thing.link_slow
subreddit = link.subreddit_slow
if (not can_view_link_comments(link) or
subreddit.type in Subreddit.private_types):
raise ForbiddenError(errors.COMMENT_NOT_ACCESSIBLE)
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
title = _('%(author)s\'s comment from discussion "%(title)s"') % {
"author": author_name,
"title": _force_unicode(link.title),
}
else:
author_name = ""
title = ""
parent = "true" if embed_options.get('parent') else "false"
html = format_html(embeds.get_inject_template(embed_options.get('omitscript')),
media=g.media_domain,
parent=parent,
live="true" if embed_options.get('live') else "false",
created=datetime.now(g.tz).isoformat(),
comment=thing.make_permalink_slow(force_domain=True),
link=link.make_permalink_slow(force_domain=True),
title=websafe(title),
uuid=uuid1(),
)
oembed_response = dict(_OEMBED_BASE,
type="rich",
title=title,
author_name=author_name,
html=html,
)
if author_name:
oembed_response['author_url'] = make_url_https('/user/' + author_name)
return oembed_response
def _oembed_post(thing, **embed_options):
subreddit = thing.subreddit_slow
if (not can_view_link_comments(thing) or
subreddit.type in Subreddit.private_types):
raise ForbiddenError(errors.POST_NOT_ACCESSIBLE)
live = ''
if embed_options.get('live'):
time = datetime.now(g.tz).isoformat()
live = 'data-card-created="{}"'.format(time)
script = ''
if not embed_options.get('omitscript', False):
script = format_html(SCRIPT_TEMPLATE,
embedly_script=EMBEDLY_SCRIPT,
)
link_url = UrlParser(thing.make_permalink_slow(force_domain=True))
link_url.update_query(ref='share', ref_source='embed')
author_name = ""
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
html = format_html(POST_EMBED_TEMPLATE,
live_data_attr=live,
link_url=link_url.unparse(),
title=websafe(thing.title),
subreddit_url=make_url_https(subreddit.path),
subreddit_name=subreddit.name,
script=script,
)
oembed_response = dict(_OEMBED_BASE,
type="rich",
title=thing.title,
author_name=author_name,
html=html,
)
return oembed_response
def _oembed_comment(thing, **embed_options):
link = thing.link_slow
if not can_view_link_comments(link):
raise ForbiddenError("Cannot access this comment.")
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
title = _('%(author)s\'s comment from discussion "%(title)s"') % {
"author": author_name,
"title": link.title,
}
else:
author_name = ""
title = ""
html = format_html(
embeds.get_inject_template(),
media=g.media_domain,
parent="true" if embed_options.get('parent') else "false",
live="true" if embed_options.get('live') else "false",
created=datetime.now(g.tz).isoformat(),
comment=thing.make_permalink_slow(force_domain=True),
link=link.make_permalink_slow(force_domain=True),
title=websafe(title),
)
oembed_response = dict(
_OEMBED_BASE,
type="rich",
title=title,
author_name=author_name,
html=html,
)
if author_name:
oembed_response['author_url'] = make_url_https('/user/' + author_name)
return oembed_response
def _oembed_comment(thing, **embed_options):
link = thing.link_slow
if not can_view_link_comments(link):
raise ForbiddenError("Cannot access this comment.")
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
title = _('%(author)s\'s comment from discussion "%(title)s"') % {
"author": author_name,
"title": _force_unicode(link.title),
}
else:
author_name = ""
title = ""
html = format_html(embeds.get_inject_template(),
media=g.media_domain,
parent="true" if embed_options.get('parent') else "false",
live="true" if embed_options.get('live') else "false",
created=datetime.now(g.tz).isoformat(),
comment=thing.make_permalink_slow(force_domain=True),
link=link.make_permalink_slow(force_domain=True),
title=websafe(title),
)
oembed_response = dict(_OEMBED_BASE,
type="rich",
title=title,
author_name=author_name,
html=html,
)
if author_name:
oembed_response['author_url'] = make_url_https('/user/' + author_name)
return oembed_response
def handle_awful_failure(fail_text):
"""
Makes sure that no errors generated in the error handler percolate
up to the user unless debug is enabled.
"""
if g.debug:
import sys
s = sys.exc_info()
# reraise the original error with the original stack trace
raise s[1], None, s[2]
try:
# log the traceback, and flag the "path" as the error location
import traceback
log.write_error_summary(fail_text)
for line in traceback.format_exc().splitlines():
g.log.error(line)
return redditbroke % (make_failien_url(), websafe(fail_text))
except:
# we are doomed. Admit defeat
return "This is an error that should never occur. You win."
def handle_awful_failure(fail_text):
"""
Makes sure that no errors generated in the error handler percolate
up to the user unless debug is enabled.
"""
if g.debug:
import sys
s = sys.exc_info()
# reraise the original error with the original stack trace
raise s[1], None, s[2]
try:
# log the traceback, and flag the "path" as the error location
import traceback
log.write_error_summary(fail_text)
for line in traceback.format_exc().splitlines():
g.log.error(line)
return redditbroke % (make_failien_url(), websafe(fail_text))
except:
# we are doomed. Admit defeat
return "This is an error that should never occur. You win."
def _people(x, label, prepend=''):
num = prepend + babel.numbers.format_number(x, c.locale)
return Score.PERSON_LABEL % \
dict(num=num, persons=websafe(label(x)))
def POST_edit_promo(self, form, jquery, username, l, title, url, selftext,
kind, disable_comments, sendreplies, media_url,
media_autoplay, media_override, gifts_embed_url,
media_url_type, domain_override, is_managed):
should_ratelimit = False
if not c.user_is_sponsor:
should_ratelimit = True
if not should_ratelimit:
c.errors.remove((errors.RATELIMIT, 'ratelimit'))
# check for user override
if not l and c.user_is_sponsor and username:
try:
user = Account._by_name(username)
except NotFound:
c.errors.add(errors.USER_DOESNT_EXIST, field="username")
form.set_error(errors.USER_DOESNT_EXIST, "username")
return
if not user.email:
c.errors.add(errors.NO_EMAIL_FOR_USER, field="username")
form.set_error(errors.NO_EMAIL_FOR_USER, "username")
return
if not user.email_verified:
c.errors.add(errors.NO_VERIFIED_EMAIL, field="username")
form.set_error(errors.NO_VERIFIED_EMAIL, "username")
return
else:
user = c.user
# check for shame banned domains
if form.has_errors("url", errors.DOMAIN_BANNED):
g.stats.simple_event('spam.shame.link')
return
# demangle URL in canonical way
if url:
if isinstance(url, (unicode, str)):
form.set_inputs(url=url)
elif isinstance(url, tuple) or isinstance(url[0], Link):
# there's already one or more links with this URL, but
# we're allowing mutliple submissions, so we really just
# want the URL
url = url[0].url
if kind == 'link':
if form.has_errors('url', errors.NO_URL, errors.BAD_URL):
return
# users can change the disable_comments on promoted links
if ((not l or not promote.is_promoted(l))
and (form.has_errors('title', errors.NO_TEXT, errors.TOO_LONG)
or jquery.has_errors('ratelimit', errors.RATELIMIT))):
return
if kind == 'self' and form.has_errors('text', errors.TOO_LONG):
return
if not l:
# creating a new promoted link
l = promote.new_promotion(title, url if kind == 'link' else 'self',
selftext if kind == 'self' else '', user,
request.ip)
l.domain_override = domain_override or None
if c.user_is_sponsor:
l.managed_promo = is_managed
l._commit()
form.redirect(promote.promo_edit_url(l))
elif not promote.is_promo(l):
return
# changing link type is not allowed
if ((l.is_self and kind == 'link')
or (not l.is_self and kind == 'self')):
c.errors.add(errors.NO_CHANGE_KIND, field="kind")
form.set_error(errors.NO_CHANGE_KIND, "kind")
return
changed = False
# live items can only be changed by a sponsor, and also
# pay the cost of de-approving the link
if not promote.is_promoted(l) or c.user_is_sponsor:
if title and title != l.title:
l.title = title
changed = not c.user_is_sponsor
if kind == 'link' and url and url != l.url:
l.url = url
changed = not c.user_is_sponsor
# only trips if the title and url are changed by a non-sponsor
if changed:
promote.unapprove_promotion(l)
# selftext can be changed at any time
if kind == 'self':
l.selftext = selftext
# comment disabling and sendreplies is free to be changed any time.
l.disable_comments = disable_comments
l.sendreplies = sendreplies
if c.user_is_sponsor:
if (form.has_errors("media_url", errors.BAD_URL)
or form.has_errors("gifts_embed_url", errors.BAD_URL)):
return
scraper_embed = media_url_type == "scrape"
media_url = media_url or None
gifts_embed_url = gifts_embed_url or None
if c.user_is_sponsor and scraper_embed and media_url != l.media_url:
if media_url:
media = _scrape_media(media_url,
autoplay=media_autoplay,
save_thumbnail=False,
use_cache=True)
if media:
l.set_media_object(media.media_object)
l.set_secure_media_object(media.secure_media_object)
l.media_url = media_url
l.gifts_embed_url = None
l.media_autoplay = media_autoplay
else:
c.errors.add(errors.SCRAPER_ERROR, field="media_url")
form.set_error(errors.SCRAPER_ERROR, "media_url")
return
else:
l.set_media_object(None)
l.set_secure_media_object(None)
l.media_url = None
l.gifts_embed_url = None
l.media_autoplay = False
if (c.user_is_sponsor and not scraper_embed
and gifts_embed_url != l.gifts_embed_url):
if gifts_embed_url:
parsed = UrlParser(gifts_embed_url)
if not is_subdomain(parsed.hostname, "redditgifts.com"):
c.errors.add(errors.BAD_URL, field="gifts_embed_url")
form.set_error(errors.BAD_URL, "gifts_embed_url")
return
iframe = """
<iframe class="redditgifts-embed"
src="%(embed_url)s"
width="710" height="500" scrolling="no"
frameborder="0" allowfullscreen>
</iframe>
""" % {
'embed_url': websafe(gifts_embed_url)
}
media_object = {
'oembed': {
'description': 'redditgifts embed',
'height': 500,
'html': iframe,
'provider_name': 'redditgifts',
'provider_url': 'http://www.redditgifts.com/',
'title': 'redditgifts secret santa 2014',
'type': 'rich',
'width': 710
},
'type': 'redditgifts'
}
l.set_media_object(media_object)
l.set_secure_media_object(media_object)
l.media_url = None
l.gifts_embed_url = gifts_embed_url
l.media_autoplay = False
else:
l.set_media_object(None)
l.set_secure_media_object(None)
l.media_url = None
l.gifts_embed_url = None
l.media_autoplay = False
if c.user_is_sponsor:
l.media_override = media_override
l.domain_override = domain_override or None
l.managed_promo = is_managed
l._commit()
form.redirect(promote.promo_edit_url(l))
def ip_span(ip):
ip = websafe(ip)
return '<!-- %s -->' % ip
def POST_promote_note(self, form, jquery, link, note):
if promote.is_promo(link):
text = PromotionLog.add(link, note)
form.find(".notes").children(":last").after("<p>" + websafe(text) +
"</p>")
def ip_span(ip):
ip = websafe(ip)
return '<!-- %s -->' % ip
def _people(x, label, prepend=''):
num = prepend + babel.numbers.format_number(x, c.locale)
return Score.PERSON_LABEL % \
dict(num=num, persons=websafe(label(x)))
def POST_promote_note(self, form, jquery, link, note):
if promote.is_promo(link):
text = PromotionLog.add(link, note)
form.find(".notes").children(":last").after(
"<p>" + websafe(text) + "</p>")
def _edit_promo(self, form, jquery, username, title, url,
selftext, kind, disable_comments, sendreplies,
media_url, media_autoplay, media_override,
gifts_embed_url, media_url_type, domain_override,
is_managed, l=None, thumbnail_file=None):
should_ratelimit = False
if not c.user_is_sponsor:
should_ratelimit = True
if not should_ratelimit:
c.errors.remove((errors.RATELIMIT, 'ratelimit'))
# check for user override
if not l and c.user_is_sponsor and username:
try:
user = Account._by_name(username)
except NotFound:
c.errors.add(errors.USER_DOESNT_EXIST, field="username")
form.set_error(errors.USER_DOESNT_EXIST, "username")
return
if not user.email:
c.errors.add(errors.NO_EMAIL_FOR_USER, field="username")
form.set_error(errors.NO_EMAIL_FOR_USER, "username")
return
if not user.email_verified:
c.errors.add(errors.NO_VERIFIED_EMAIL, field="username")
form.set_error(errors.NO_VERIFIED_EMAIL, "username")
return
else:
user = c.user
# check for shame banned domains
if form.has_errors("url", errors.DOMAIN_BANNED):
g.stats.simple_event('spam.shame.link')
return
# demangle URL in canonical way
if url:
if isinstance(url, (unicode, str)):
form.set_inputs(url=url)
elif isinstance(url, tuple) or isinstance(url[0], Link):
# there's already one or more links with this URL, but
# we're allowing mutliple submissions, so we really just
# want the URL
url = url[0].url
if kind == 'link':
if form.has_errors('url', errors.NO_URL, errors.BAD_URL):
return
# users can change the disable_comments on promoted links
if ((not l or not promote.is_promoted(l)) and
(form.has_errors('title', errors.NO_TEXT, errors.TOO_LONG) or
jquery.has_errors('ratelimit', errors.RATELIMIT))):
return
if kind == 'self' and form.has_errors('text', errors.TOO_LONG):
return
if not l:
# creating a new promoted link
l = promote.new_promotion(title, url if kind == 'link' else 'self',
selftext if kind == 'self' else '',
user, request.ip)
l.domain_override = domain_override or None
if c.user_is_sponsor:
l.managed_promo = is_managed
l._commit()
# only set the thumbnail when creating a link
if thumbnail_file:
try:
force_thumbnail(l, thumbnail_file)
l._commit()
except IOError:
pass
form.redirect(promote.promo_edit_url(l))
elif not promote.is_promo(l):
return
# changing link type is not allowed
if ((l.is_self and kind == 'link') or
(not l.is_self and kind == 'self')):
c.errors.add(errors.NO_CHANGE_KIND, field="kind")
form.set_error(errors.NO_CHANGE_KIND, "kind")
return
changed = False
# live items can only be changed by a sponsor, and also
# pay the cost of de-approving the link
if not promote.is_promoted(l) or c.user_is_sponsor:
if title and title != l.title:
l.title = title
changed = not c.user_is_sponsor
if kind == 'link' and url and url != l.url:
l.url = url
changed = not c.user_is_sponsor
# only trips if the title and url are changed by a non-sponsor
if changed:
promote.unapprove_promotion(l)
# selftext can be changed at any time
if kind == 'self':
l.selftext = selftext
# comment disabling and sendreplies is free to be changed any time.
l.disable_comments = disable_comments
l.sendreplies = sendreplies
if c.user_is_sponsor:
if (form.has_errors("media_url", errors.BAD_URL) or
form.has_errors("gifts_embed_url", errors.BAD_URL)):
return
scraper_embed = media_url_type == "scrape"
media_url = media_url or None
gifts_embed_url = gifts_embed_url or None
if c.user_is_sponsor and scraper_embed and media_url != l.media_url:
if media_url:
media = _scrape_media(
media_url, autoplay=media_autoplay,
save_thumbnail=False, use_cache=True)
if media:
l.set_media_object(media.media_object)
l.set_secure_media_object(media.secure_media_object)
l.media_url = media_url
l.gifts_embed_url = None
l.media_autoplay = media_autoplay
else:
c.errors.add(errors.SCRAPER_ERROR, field="media_url")
form.set_error(errors.SCRAPER_ERROR, "media_url")
return
else:
l.set_media_object(None)
l.set_secure_media_object(None)
l.media_url = None
l.gifts_embed_url = None
l.media_autoplay = False
if (c.user_is_sponsor and not scraper_embed and
gifts_embed_url != l.gifts_embed_url):
if gifts_embed_url:
parsed = UrlParser(gifts_embed_url)
if not is_subdomain(parsed.hostname, "redditgifts.com"):
c.errors.add(errors.BAD_URL, field="gifts_embed_url")
form.set_error(errors.BAD_URL, "gifts_embed_url")
return
iframe = """
<iframe class="redditgifts-embed"
src="%(embed_url)s"
width="710" height="500" scrolling="no"
frameborder="0" allowfullscreen>
</iframe>
""" % {'embed_url': websafe(gifts_embed_url)}
media_object = {
'oembed': {
'description': 'redditgifts embed',
'height': 500,
'html': iframe,
'provider_name': 'redditgifts',
'provider_url': 'http://www.redditgifts.com/',
'title': 'redditgifts secret santa 2014',
'type': 'rich',
'width': 710},
'type': 'redditgifts'
}
l.set_media_object(media_object)
l.set_secure_media_object(media_object)
l.media_url = None
l.gifts_embed_url = gifts_embed_url
l.media_autoplay = False
else:
l.set_media_object(None)
l.set_secure_media_object(None)
l.media_url = None
l.gifts_embed_url = None
l.media_autoplay = False
if c.user_is_sponsor:
l.media_override = media_override
l.domain_override = domain_override or None
l.managed_promo = is_managed
l._commit()
form.redirect(promote.promo_edit_url(l))
def ip_span(ip):
ip = websafe(ip)
return "<!-- %s -->" % ip
