def POST_request_promo(self, srnames):
if not srnames:
return
srnames = srnames.split('+')
# request multiple ads in case some are hidden by the builder due
# to the user's hides/preferences
response = adzerk_request(srnames)
if not response:
g.stats.simple_event('adzerk.request.no_promo')
return
res_by_campaign = {r.campaign: r for r in response}
tuples = [promote.PromoTuple(r.link, 1., r.campaign) for r in response]
builder = CampaignBuilder(tuples, wrap=default_thing_wrapper(),
keep_fn=promote.promo_keep_fn,
num=1,
skip=True)
listing = LinkListing(builder, nextprev=False).listing()
if listing.things:
g.stats.simple_event('adzerk.request.valid_promo')
w = listing.things[0]
r = res_by_campaign[w.campaign]
up = UrlParser(r.imp_pixel)
up.hostname = "pixel.redditmedia.com"
w.adserver_imp_pixel = up.unparse()
w.adserver_click_url = r.click_url
w.num = ""
return spaceCompress(w.render())
else:
g.stats.simple_event('adzerk.request.skip_promo')
def process_message(msgs, chan):
"""Update get_domain_links(), the Links by domain precomputed query.
get_domain_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by domain allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from r2.lib.db.queries import add_queries, get_domain_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links,)
links_by_domain = defaultdict(list)
for link in links:
parsed = UrlParser(link.url)
# update the listings for all permutations of the link's domain
for domain in parsed.domain_permutations():
links_by_domain[domain].append(link)
for d, links in links_by_domain.iteritems():
with g.stats.get_timer("link_vote_processor.domain_queries"):
add_queries(
queries=[
get_domain_links(d, sort, "all") for sort in SORTS],
insert_items=links,
)
def GET_framebuster(self, what = None, blah = None):
"""
renders the contents of the iframe which, on a cname, checks
if the user is currently logged into reddit.
if this page is hit from the primary domain, redirects to the
cnamed domain version of the site. If the user is logged in,
this cnamed version will drop a boolean session cookie on that
domain so that subsequent page reloads will be caught in
middleware and a frame will be inserted around the content.
If the user is not logged in, previous session cookies will be
emptied so that subsequent refreshes will not be rendered in
that pesky frame.
"""
if not c.site.domain:
return ""
elif c.cname:
return FrameBuster(login = (what == "login")).render()
else:
path = "/framebuster/"
if c.user_is_loggedin:
path += "login/"
u = UrlParser(path + str(random.random()))
u.mk_cname(require_frame = False, subreddit = c.site,
port = request.port)
return self.redirect(u.unparse())
# the user is not logged in or there is no cname.
return FrameBuster(login = False).render()
def purge_url(self, url):
"""Purge an image (by url) from imgix.
Reference: http://www.imgix.com/docs/tutorials/purging-images
Note that as mentioned in the imgix docs, in order to remove
an image, this function should be used *after* already
removing the image from our source, or imgix will just re-fetch
and replace the image with a new copy even after purging.
"""
p = UrlParser(url)
if p.hostname == g.imgix_domain:
p.hostname = g.imgix_purge_domain
elif p.hostname == g.imgix_gif_domain:
p.hostname = g.imgix_gif_purge_domain
url = p.unparse()
requests.post(
"https://api.imgix.com/v2/image/purger",
auth=(g.secrets["imgix_api_key"], ""),
data={"url": url},
)
def format_output_url(cls, url, **kw):
"""
Helper method used during redirect to ensure that the redirect
url (assisted by frame busting code or javasctipt) will point
to the correct domain and not have any extra dangling get
parameters. The extensions are also made to match and the
resulting url is utf8 encoded.
Node: for development purposes, also checks that the port
matches the request port
"""
preserve_extension = kw.pop("preserve_extension", True)
u = UrlParser(url)
if u.is_reddit_url():
# make sure to pass the port along if not 80
if not kw.has_key('port'):
kw['port'] = request.port
# make sure the extensions agree with the current page
if preserve_extension and c.extension:
u.set_extension(c.extension)
# unparse and encode it un utf8
rv = _force_unicode(u.unparse()).encode('utf8')
if "\n" in rv or "\r" in rv:
abort(400)
return rv
def url_for_title(self, title):
"""Uses the MediaWiki API to get the URL for a wiki page
with the given title"""
if title is None:
return None
from pylons import g
cache_key = ('wiki_url_%s' % title).encode('ascii', 'ignore')
wiki_url = g.cache.get(cache_key)
if wiki_url is None:
# http://www.mediawiki.org/wiki/API:Query_-_Properties#info_.2F_in
api = UrlParser(g.wiki_api_url)
api.update_query(
action = 'query',
titles= title,
prop = 'info',
format = 'yaml',
inprop = 'url'
)
try:
response = urlopen(api.unparse()).read()
parsed_response = yaml.load(response, Loader=yaml.CLoader)
page = parsed_response['query']['pages'][0]
except:
return None
wiki_url = page.get('fullurl').strip()
# Things are created every couple of days so 12 hours seems
# to be a reasonable cache time
g.permacache.set(cache_key, wiki_url, time=3600 * 12)
return wiki_url
def _update_redirect_uri(base_redirect_uri, params, as_fragment=False):
parsed = UrlParser(base_redirect_uri)
if as_fragment:
parsed.fragment = urlencode(params)
else:
parsed.update_query(**params)
return parsed.unparse()
def allowed_media_preview_url(url):
p = UrlParser(url)
if p.has_static_image_extension():
return True
for allowed_domain in g.media_preview_domain_whitelist:
if is_subdomain(p.hostname, allowed_domain):
return True
return False
def GET_framebuster(self):
if c.site.domain and c.user_is_loggedin:
u = UrlParser(c.site.path + "/frame")
u.put_in_frame()
c.cname = True
return self.redirect(u.unparse())
return "fail"
def _key_from_url(cls, url):
if not utils.domain(url) in g.case_sensitive_domains:
keyurl = _force_utf8(UrlParser.base_url(url.lower()))
else:
# Convert only hostname to lowercase
up = UrlParser(url)
up.hostname = up.hostname.lower()
keyurl = _force_utf8(UrlParser.base_url(up.unparse()))
return keyurl
def redirect_to_host(hostname, path=None):
"""Redirect (302) to the specified path and host."""
if path is None:
path = request.path
u = UrlParser(path)
u.hostname = hostname
# 307 redirect so request method is retained
abort(307, location=u.unparse())
def POST_request_promo(self, srnames, is_mobile_web, platform, loid, is_refresh):
self.OPTIONS_request_promo()
if not srnames:
return
# backwards compat
if platform is None:
platform = "mobile_web" if is_mobile_web else "desktop"
srnames = srnames.split('+')
# request multiple ads in case some are hidden by the builder due
# to the user's hides/preferences
response = adzerk_request(srnames, self.get_uid(loid),
platform=platform)
if not response:
g.stats.simple_event('adzerk.request.no_promo')
return
# for adservers, adzerk returns markup so we pass it to the client
if isinstance(response, AdserverResponse):
g.stats.simple_event('adzerk.request.adserver')
return responsive(response.body)
res_by_campaign = {r.campaign: r for r in response}
adserver_click_urls = {r.campaign: r.click_url for r in response}
tuples = [promote.PromoTuple(r.link, 1., r.campaign) for r in response]
builder = CampaignBuilder(tuples, wrap=default_thing_wrapper(),
keep_fn=promote.promo_keep_fn,
num=1,
skip=True)
listing = LinkListing(builder, nextprev=False).listing()
promote.add_trackers(listing.things, c.site, adserver_click_urls=adserver_click_urls)
promote.update_served(listing.things)
if listing.things:
g.stats.simple_event('adzerk.request.valid_promo')
if is_refresh:
g.stats.simple_event('adzerk.request.auto_refresh')
w = listing.things[0]
r = res_by_campaign[w.campaign]
up = UrlParser(r.imp_pixel)
up.hostname = "pixel.redditmedia.com"
w.adserver_imp_pixel = up.unparse()
w.adserver_upvote_pixel = r.upvote_pixel
w.adserver_downvote_pixel = r.downvote_pixel
w.adserver_click_url = r.click_url
w.num = ""
return responsive(w.render(), space_compress=True)
else:
g.stats.simple_event('adzerk.request.skip_promo')
def make_scraper(url):
parsed = UrlParser(url)
if parsed.is_reddit_url():
if parsed.path.startswith("/live/"):
try:
event_id = parsed.path.split("/")[2]
except IndexError:
return
else:
return _LiveUpdateScraper(event_id)
def test_sign_url(self):
u = UrlParser('http://examples.imgix.net/frog.jpg?w=100')
signed_url = self.provider._sign_url(u, 'abcdef')
self.assertEqual(signed_url.unparse(),
'http://examples.imgix.net/frog.jpg?w=100&s=cd3bdf071108af73b15c21bdcee5e49c')
u = UrlParser('http://examples.imgix.net/frog.jpg')
u.update_query(w=100)
signed_url = self.provider._sign_url(u, 'abcdef')
self.assertEqual(signed_url.unparse(),
'http://examples.imgix.net/frog.jpg?w=100&s=cd3bdf071108af73b15c21bdcee5e49c')
def add_to_domain_query_q(link):
parsed = UrlParser(link.url)
if not parsed.domain_permutations():
# no valid domains found
return
if g.shard_domain_query_queues:
domain_shard = hash(parsed.hostname) % 10
queue_name = "domain_query_%s_q" % domain_shard
else:
queue_name = "domain_query_q"
amqp.add_item(queue_name, link._fullname)
def run(self, url):
if not url:
return None
u = UrlParser(url)
# TODO: We should probably set error messages in these cases.
if not u.is_reddit_url():
return None
event_id = re.match(r'/live/(\w+)/?', u.path)
if not event_id:
return None
return VLiveUpdateEvent.run(self, event_id.group(1))
def _get_scrape_url(link):
if not link.is_self:
sr_name = link.subreddit_slow.name
if not feature.is_enabled("imgur_gif_conversion", subreddit=sr_name):
return link.url
p = UrlParser(link.url)
# If it's a gif link on imgur, replacing it with gifv should
# give us the embedly friendly video url
if is_subdomain(p.hostname, "imgur.com"):
if p.path_extension().lower() == "gif":
p.set_extension("gifv")
return p.unparse()
return link.url
urls = extract_urls_from_markdown(link.selftext)
second_choice = None
for url in urls:
p = UrlParser(url)
if p.is_reddit_url():
continue
# If we don't find anything we like better, use the first image.
if not second_choice:
second_choice = url
# This is an optimization for "proof images" in AMAs.
if is_subdomain(p.netloc, 'imgur.com') or p.has_image_extension():
return url
return second_choice
def test_default_prefix(self):
u = UrlParser('http://i.reddit.com/r/redditdev')
u.switch_subdomain_by_extension()
self.assertEquals('http://www.reddit.com/r/redditdev', u.unparse())
u = UrlParser('http://i.reddit.com/r/redditdev')
u.switch_subdomain_by_extension('does-not-exist')
self.assertEquals('http://www.reddit.com/r/redditdev', u.unparse())
def GET_link_id_redirect(self, link):
if not link:
abort(404)
elif not link.subreddit_slow.can_view(c.user):
# don't disclose the subreddit/title of a post via the redirect url
abort(403)
else:
redirect_url = link.make_permalink_slow(force_domain=True)
query_params = dict(request.GET)
if query_params:
url = UrlParser(redirect_url)
url.update_query(**query_params)
redirect_url = url.unparse()
return self.redirect(redirect_url, code=301)
def by_url_key(cls, url):
maxlen = 250
template = 'byurl(%s,%s)'
keyurl = _force_utf8(UrlParser.base_url(url.lower()))
hexdigest = md5(keyurl).hexdigest()
usable_len = maxlen-len(template)-len(hexdigest)
return template % (hexdigest, keyurl[:usable_len])
def run(self, url):
if not url:
return None
u = UrlParser(url)
# TODO: We should probably set error messages in these cases.
if not u.is_reddit_url():
return None
event_id = re.match(r'/live/(\w+)/?', u.path)
if not event_id:
return None
try:
return models.LiveUpdateEvent._byID(event_id.group(1))
except tdb_cassandra.NotFound:
return None
def test_normal_urls(self):
u = UrlParser('http://www.reddit.com/r/redditdev')
u.switch_subdomain_by_extension('compact')
result = u.unparse()
self.assertEquals('http://i.reddit.com/r/redditdev', result)
u = UrlParser(result)
u.switch_subdomain_by_extension('mobile')
result = u.unparse()
self.assertEquals('http://m.reddit.com/r/redditdev', result)
def add_sr(path, sr_path = True, nocname=False, force_hostname = False):
"""
Given a path (which may be a full-fledged url or a relative path),
parses the path and updates it to include the subreddit path
according to the rules set by its arguments:
* force_hostname: if True, force the url's hotname to be updated
even if it is already set in the path, and subject to the
c.cname/nocname combination. If false, the path will still
have its domain updated if no hostname is specified in the url.
* nocname: when updating the hostname, overrides the value of
c.cname to set the hotname to g.domain. The default behavior
is to set the hostname consistent with c.cname.
* sr_path: if a cname is not used for the domain, updates the
path to include c.site.path.
"""
u = UrlParser(path)
if sr_path and (nocname or not c.cname):
u.path_add_subreddit(c.site)
if not u.hostname or force_hostname:
u.hostname = get_domain(cname = (c.cname and not nocname),
subreddit = False)
if c.render_style == 'mobile':
u.set_extension('mobile')
return u.unparse()
def format_output_url(cls, url, **kw):
"""
Helper method used during redirect to ensure that the redirect
url (assisted by frame busting code or javasctipt) will point
to the correct domain and not have any extra dangling get
parameters. The extensions are also made to match and the
resulting url is utf8 encoded.
Node: for development purposes, also checks that the port
matches the request port
"""
u = UrlParser(url)
if u.is_reddit_url():
# make sure to pass the port along if not 80
if not kw.has_key('port'):
kw['port'] = request.port
# disentagle the cname (for urls that would have
# cnameframe=1 in them)
u.mk_cname(**kw)
# make sure the extensions agree with the current page
if c.extension:
u.set_extension(c.extension)
# unparse and encode it un utf8
return _force_unicode(u.unparse()).encode('utf8')
def _oembed_post(thing, **embed_options):
subreddit = thing.subreddit_slow
if (not can_view_link_comments(thing) or
subreddit.type in Subreddit.private_types):
raise ForbiddenError(errors.POST_NOT_ACCESSIBLE)
live = ''
if embed_options.get('live'):
time = datetime.now(g.tz).isoformat()
live = 'data-card-created="{}"'.format(time)
script = ''
if not embed_options.get('omitscript', False):
script = format_html(SCRIPT_TEMPLATE,
embedly_script=EMBEDLY_SCRIPT,
)
link_url = UrlParser(thing.make_permalink_slow(force_domain=True))
link_url.update_query(ref='share', ref_source='embed')
author_name = ""
if not thing._deleted:
author = thing.author_slow
if author._deleted:
author_name = _("[account deleted]")
else:
author_name = author.name
html = format_html(POST_EMBED_TEMPLATE,
live_data_attr=live,
link_url=link_url.unparse(),
title=websafe(thing.title),
subreddit_url=make_url_https(subreddit.path),
subreddit_name=subreddit.name,
script=script,
)
oembed_response = dict(_OEMBED_BASE,
type="rich",
title=thing.title,
author_name=author_name,
html=html,
)
return oembed_response
def test_same_url(self):
u = UrlParser('http://example.com:8000/a;b?foo=bar&bar=baz#spam')
u2 = UrlParser('http://example.com:8000/a;b?bar=baz&foo=bar#spam')
self.assertEquals(u, u2)
u3 = UrlParser('')
u3.scheme = 'http'
u3.hostname = 'example.com'
u3.port = 8000
u3.path = '/a'
u3.params = 'b'
u3.update_query(foo='bar', bar='baz')
u3.fragment = 'spam'
self.assertEquals(u, u3)
def _get_scrape_url(link):
if not link.is_self:
return link.url
urls = extract_urls_from_markdown(link.selftext)
second_choice = None
for url in urls:
p = UrlParser(url)
if p.is_reddit_url():
continue
# If we don't find anything we like better, use the first image.
if not second_choice:
second_choice = url
# This is an optimization for "proof images" in AMAs.
if is_subdomain(p.netloc, 'imgur.com') or p.has_image_extension():
return url
return second_choice
def POST_bpoptions(self, all_langs, **prefs):
u = UrlParser(c.site.path + "prefs")
bpfilter_prefs(prefs, c.user)
if c.errors.errors:
for error in c.errors.errors:
if error[1] == 'stylesheet_override':
u.update_query(error_style_override=error[0])
else:
u.update_query(generic_error=error[0])
return self.redirect(u.unparse())
set_prefs(c.user, prefs)
c.user._commit()
u.update_query(done='true')
return self.redirect(u.unparse())
def GET_quarantine(self, dest):
if not feature.is_enabled('quarantine'):
return self.abort404()
sr = UrlParser(dest).get_subreddit()
# if dest doesn't include a quarantined subreddit,
# redirect to the homepage or the original destination
if not sr:
return self.redirect('/')
elif isinstance(sr, FakeSubreddit) or sr.is_exposed(c.user):
return self.redirect(dest)
return InterstitialPage(
_("quarantined"),
content=QuarantineInterstitial(
sr_name=sr.name,
logged_in=c.user_is_loggedin,
email_verified=c.user_is_loggedin and c.user.email_verified,
),
).render()
def GET_quarantine(self, dest):
sr = UrlParser(dest).get_subreddit()
# if dest doesn't include a quarantined subreddit,
# redirect to the homepage or the original destination
if not sr:
return self.redirect('/')
elif isinstance(sr, FakeSubreddit) or sr.is_exposed(c.user):
return self.redirect(dest)
errpage = InterstitialPage(
_("quarantined"),
content=QuarantineInterstitial(
sr_name=sr.name,
logged_in=c.user_is_loggedin,
email_verified=c.user_is_loggedin and c.user.email_verified,
),
)
request.environ['usable_error_content'] = errpage.render()
self.abort403()
def POST_options(self, all_langs, pref_lang, **kw):
#temporary. eventually we'll change pref_clickgadget to an
#integer preference
kw['pref_clickgadget'] = kw['pref_clickgadget'] and 5 or 0
if c.user.pref_show_promote is None:
kw['pref_show_promote'] = None
elif not kw.get('pref_show_promote'):
kw['pref_show_promote'] = False
if not kw.get("pref_over_18") or not c.user.pref_over_18:
kw['pref_no_profanity'] = True
if kw.get("pref_no_profanity") or c.user.pref_no_profanity:
kw['pref_label_nsfw'] = True
self.set_options(all_langs, pref_lang, **kw)
u = UrlParser(c.site.path + "prefs")
u.update_query(done='true')
if c.cname:
u.put_in_frame()
return self.redirect(u.unparse())
def process(thing):
if thing.deleted:
return
thing_cls = thingcls_by_name[thing.thing_type]
fname = make_fullname(thing_cls, thing.thing_id)
thing_score = score(thing.ups, thing.downs)
thing_controversy = controversy(thing.ups, thing.downs)
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
yield ("user/%s/top/%s/%d" %
(thing.thing_type, interval, thing.author_id), thing_score,
thing.timestamp, fname)
yield ("user/%s/controversial/%s/%d" %
(thing.thing_type, interval, thing.author_id),
thing_controversy, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
thing_score, thing.timestamp, fname)
yield ("sr/link/controversial/%s/%d" % (interval, thing.sr_id),
thing_controversy, thing.timestamp, fname)
if thing.url:
for domain in UrlParser(thing.url).domain_permutations():
yield ("domain/link/top/%s/%s" % (interval, domain),
thing_score, thing.timestamp, fname)
yield ("domain/link/controversial/%s/%s" %
(interval, domain), thing_controversy,
thing.timestamp, fname)
def _update_redirect_uri(base_redirect_uri, params):
parsed = UrlParser(base_redirect_uri)
parsed.update_query(**params)
return parsed.unparse()
def __init__(self, url):
self.url = url
# Having the source document's protocol on hand makes it easier to deal
# with protocol-relative urls we extract from it.
self.protocol = UrlParser(url).scheme
def add_sr(path,
sr_path=True,
nocname=False,
force_hostname=False,
retain_extension=True):
"""
Given a path (which may be a full-fledged url or a relative path),
parses the path and updates it to include the subreddit path
according to the rules set by its arguments:
* force_hostname: if True, force the url's hostname to be updated
even if it is already set in the path, and subject to the
c.cname/nocname combination. If false, the path will still
have its domain updated if no hostname is specified in the url.
* nocname: when updating the hostname, overrides the value of
c.cname to set the hostname to g.domain. The default behavior
is to set the hostname consistent with c.cname.
* sr_path: if a cname is not used for the domain, updates the
path to include c.site.path.
For caching purposes: note that this function uses:
c.cname, c.render_style, c.site.name
"""
# don't do anything if it is just an anchor
if path.startswith(('#', 'javascript:')):
return path
u = UrlParser(path)
if sr_path and (nocname or not c.cname):
u.path_add_subreddit(c.site)
if not u.hostname or force_hostname:
if c.secure:
u.hostname = request.host
else:
u.hostname = get_domain(cname=(c.cname and not nocname),
subreddit=False)
if c.secure:
u.scheme = "https"
if retain_extension:
if c.render_style == 'mobile':
u.set_extension('mobile')
elif c.render_style == 'compact':
u.set_extension('compact')
return u.unparse()
def test_empty_extension(self):
u = UrlParser('http://example.com/a.')
self.assertEquals('', u.path_extension())
def test_same_url(self):
u = UrlParser('http://example.com:8000/a;b?foo=bar&bar=baz#spam')
u2 = UrlParser('http://example.com:8000/a;b?bar=baz&foo=bar#spam')
self.assertEquals(u, u2)
u3 = UrlParser('')
u3.scheme = 'http'
u3.hostname = 'example.com'
u3.port = 8000
u3.path = '/a'
u3.params = 'b'
u3.update_query(foo='bar', bar='baz')
u3.fragment = 'spam'
self.assertEquals(u, u3)
def make_feedurl(user, path, ext="rss"):
u = UrlParser(path)
u.update_query(user=user.name, feed=make_feedhash(user, path))
u.set_extension(ext)
return u.unparse()
def test_different_domains(self):
u = UrlParser('http://example.com')
u2 = UrlParser('http://example.org')
self.assertNotEquals(u, u2)
def test_unicode_query_params(self):
u = UrlParser(u'http://example.com/?page=ｕｎｉｃｏｄｅ：（')
u2 = UrlParser('http://example.com/')
u2.update_query(page=u'ｕｎｉｃｏｄｅ：（')
self.assertEquals(u, u2)
def test_different_fragments(self):
u = UrlParser('http://example.com/')
u2 = UrlParser('http://example.com/#foo')
u3 = UrlParser('http://example.com/#bar')
self.assertNotEquals(u, u2)
self.assertNotEquals(u2, u3)
def test_different_protocols(self):
u = UrlParser('http://example.com')
u2 = UrlParser('https://example.com')
self.assertNotEquals(u, u2)
def test_different_paths(self):
u = UrlParser('http://example.com')
u2 = UrlParser('http://example.com/a')
u3 = UrlParser('http://example.com/b')
self.assertNotEquals(u, u2)
self.assertNotEquals(u2, u3)
def _is_safe_reddit_url(self, url, subreddit=None):
web_safe = UrlParser(url).is_web_safe_url()
return web_safe and UrlParser(url).is_reddit_url(subreddit)
def _key_from_url(cls, url):
keyurl = _force_utf8(UrlParser.base_url(url.lower()))
return keyurl
def test_nested_file(self):
u = UrlParser('http://example.com/foo/a.jpg')
self.assertEquals('jpg', u.path_extension())
def add_sr(path,
sr_path=True,
nocname=False,
force_hostname=False,
retain_extension=True,
force_https=False,
force_extension=None):
"""
Given a path (which may be a full-fledged url or a relative path),
parses the path and updates it to include the subreddit path
according to the rules set by its arguments:
* sr_path: if a cname is not used for the domain, updates the
path to include c.site.path.
* nocname: deprecated.
* force_hostname: if True, force the url's hostname to be updated
even if it is already set in the path. If false, the path will still
have its domain updated if no hostname is specified in the url.
* retain_extension: if True, sets the extention according to
c.render_style.
* force_https: force the URL scheme to https
For caching purposes: note that this function uses:
c.render_style, c.site.name
"""
# don't do anything if it is just an anchor
if path.startswith(('#', 'javascript:')):
return path
u = UrlParser(path)
if sr_path:
u.path_add_subreddit(c.site)
if not u.hostname or force_hostname:
u.hostname = get_domain(subreddit=False)
if (c.secure and u.is_reddit_url()) or force_https:
u.scheme = "https"
if force_extension is not None:
u.set_extension(force_extension)
elif retain_extension:
if c.render_style == 'mobile':
u.set_extension('mobile')
elif c.render_style == 'compact':
u.set_extension('compact')
return u.unparse()
def add_ext_to_link(link):
url = UrlParser(link.get('href'))
if url.is_reddit_url():
link['href'] = add_sr(link.get('href'), sr_path=False)
def update_query(base_url, **kw):
parsed = UrlParser(base_url)
parsed.update_query(**kw)
return parsed.unparse()
def test_only_extension(self):
u = UrlParser('http://example.com/.bashrc')
self.assertEquals('bashrc', u.path_extension())
def test_two_extensions(self):
u = UrlParser('http://example.com/a.jpg.exe')
self.assertEquals('exe', u.path_extension())
def test_different_ports(self):
u = UrlParser('http://example.com')
u2 = UrlParser('http://example.com:8000')
u3 = UrlParser('http://example.com:8008')
self.assertNotEquals(u, u2)
self.assertNotEquals(u2, u3)
def test_integer_query_params(self):
u = UrlParser('http://example.com/?page=1234')
u2 = UrlParser('http://example.com/')
u2.update_query(page=1234)
self.assertEquals(u, u2)
def test_different_params(self):
u = UrlParser('http://example.com/')
u2 = UrlParser('http://example.com/;foo')
u3 = UrlParser('http://example.com/;bar')
self.assertNotEquals(u, u2)
self.assertNotEquals(u2, u3)
def resize_image(self,
image,
width=None,
censor_nsfw=False,
max_ratio=None):
url = UrlParser(image['url'])
url.hostname = g.imgix_domain
# Let's encourage HTTPS; it's cool, works just fine on HTTP pages, and
# will prevent insecure content warnings on HTTPS pages.
url.scheme = 'https'
if max_ratio:
url.update_query(fit='crop')
# http://www.imgix.com/docs/reference/size#param-crop
url.update_query(crop='faces,entropy')
url.update_query(arh=max_ratio)
if width:
if width > image['width']:
raise NotLargeEnough()
# http://www.imgix.com/docs/reference/size#param-w
url.update_query(w=width)
if censor_nsfw:
# Do an initial blur to make sure we're getting rid of icky
# details.
#
# http://www.imgix.com/docs/reference/stylize#param-blur
url.update_query(blur=600)
# And then add pixellation to help the image compress well.
#
# http://www.imgix.com/docs/reference/stylize#param-px
url.update_query(px=32)
if g.imgix_signing:
url = self._sign_url(url, g.secrets['imgix_signing_token'])
return url.unparse()
def POST_edit_promo(self, form, jquery, username, l, title, url, selftext,
kind, disable_comments, sendreplies, media_url,
media_autoplay, media_override, gifts_embed_url,
media_url_type, domain_override, is_managed):
should_ratelimit = False
if not c.user_is_sponsor:
should_ratelimit = True
if not should_ratelimit:
c.errors.remove((errors.RATELIMIT, 'ratelimit'))
# check for user override
if not l and c.user_is_sponsor and username:
try:
user = Account._by_name(username)
except NotFound:
c.errors.add(errors.USER_DOESNT_EXIST, field="username")
form.set_error(errors.USER_DOESNT_EXIST, "username")
return
if not user.email:
c.errors.add(errors.NO_EMAIL_FOR_USER, field="username")
form.set_error(errors.NO_EMAIL_FOR_USER, "username")
return
if not user.email_verified:
c.errors.add(errors.NO_VERIFIED_EMAIL, field="username")
form.set_error(errors.NO_VERIFIED_EMAIL, "username")
return
else:
user = c.user
# check for shame banned domains
if form.has_errors("url", errors.DOMAIN_BANNED):
g.stats.simple_event('spam.shame.link')
return
# demangle URL in canonical way
if url:
if isinstance(url, (unicode, str)):
form.set_inputs(url=url)
elif isinstance(url, tuple) or isinstance(url[0], Link):
# there's already one or more links with this URL, but
# we're allowing mutliple submissions, so we really just
# want the URL
url = url[0].url
if kind == 'link':
if form.has_errors('url', errors.NO_URL, errors.BAD_URL):
return
# users can change the disable_comments on promoted links
if ((not l or not promote.is_promoted(l))
and (form.has_errors('title', errors.NO_TEXT, errors.TOO_LONG)
or jquery.has_errors('ratelimit', errors.RATELIMIT))):
return
if kind == 'self' and form.has_errors('text', errors.TOO_LONG):
return
if not l:
# creating a new promoted link
l = promote.new_promotion(title, url if kind == 'link' else 'self',
selftext if kind == 'self' else '', user,
request.ip)
l.domain_override = domain_override or None
if c.user_is_sponsor:
l.managed_promo = is_managed
l._commit()
form.redirect(promote.promo_edit_url(l))
elif not promote.is_promo(l):
return
# changing link type is not allowed
if ((l.is_self and kind == 'link')
or (not l.is_self and kind == 'self')):
c.errors.add(errors.NO_CHANGE_KIND, field="kind")
form.set_error(errors.NO_CHANGE_KIND, "kind")
return
changed = False
# live items can only be changed by a sponsor, and also
# pay the cost of de-approving the link
if not promote.is_promoted(l) or c.user_is_sponsor:
if title and title != l.title:
l.title = title
changed = not c.user_is_sponsor
if kind == 'link' and url and url != l.url:
l.url = url
changed = not c.user_is_sponsor
# only trips if the title and url are changed by a non-sponsor
if changed:
promote.unapprove_promotion(l)
# selftext can be changed at any time
if kind == 'self':
l.selftext = selftext
# comment disabling and sendreplies is free to be changed any time.
l.disable_comments = disable_comments
l.sendreplies = sendreplies
if c.user_is_sponsor:
if (form.has_errors("media_url", errors.BAD_URL)
or form.has_errors("gifts_embed_url", errors.BAD_URL)):
return
scraper_embed = media_url_type == "scrape"
media_url = media_url or None
gifts_embed_url = gifts_embed_url or None
if c.user_is_sponsor and scraper_embed and media_url != l.media_url:
if media_url:
media = _scrape_media(media_url,
autoplay=media_autoplay,
save_thumbnail=False,
use_cache=True)
if media:
l.set_media_object(media.media_object)
l.set_secure_media_object(media.secure_media_object)
l.media_url = media_url
l.gifts_embed_url = None
l.media_autoplay = media_autoplay
else:
c.errors.add(errors.SCRAPER_ERROR, field="media_url")
form.set_error(errors.SCRAPER_ERROR, "media_url")
return
else:
l.set_media_object(None)
l.set_secure_media_object(None)
l.media_url = None
l.gifts_embed_url = None
l.media_autoplay = False
if (c.user_is_sponsor and not scraper_embed
and gifts_embed_url != l.gifts_embed_url):
if gifts_embed_url:
parsed = UrlParser(gifts_embed_url)
if not is_subdomain(parsed.hostname, "redditgifts.com"):
c.errors.add(errors.BAD_URL, field="gifts_embed_url")
form.set_error(errors.BAD_URL, "gifts_embed_url")
return
iframe = """
<iframe class="redditgifts-embed"
src="%(embed_url)s"
width="710" height="500" scrolling="no"
frameborder="0" allowfullscreen>
</iframe>
""" % {
'embed_url': websafe(gifts_embed_url)
}
media_object = {
'oembed': {
'description': 'redditgifts embed',
'height': 500,
'html': iframe,
'provider_name': 'redditgifts',
'provider_url': 'http://www.redditgifts.com/',
'title': 'redditgifts secret santa 2014',
'type': 'rich',
'width': 710
},
'type': 'redditgifts'
}
l.set_media_object(media_object)
l.set_secure_media_object(media_object)
l.media_url = None
l.gifts_embed_url = gifts_embed_url
l.media_autoplay = False
else:
l.set_media_object(None)
l.set_secure_media_object(None)
l.media_url = None
l.gifts_embed_url = None
l.media_autoplay = False
if c.user_is_sponsor:
l.media_override = media_override
l.domain_override = domain_override or None
l.managed_promo = is_managed
l._commit()
form.redirect(promote.promo_edit_url(l))
def test_different_objects(self):
u = UrlParser('http://example.com')
self.assertNotEquals(u, None)
def test_different_queries(self):
u = UrlParser('http://example.com/')
u2 = UrlParser('http://example.com/?foo')
u3 = UrlParser('http://example.com/?foo=bar')
self.assertNotEquals(u, u2)
self.assertNotEquals(u2, u3)
Field('contents', 'title', tokenize=True),
Field(
'boost',
lambda t: int(t._hot * 1000),
# yes, it's a copy of 'hot'
is_number=True,
reverse=True),
Field('author_id'),
ThingField('author', Account, 'author_id', 'name'),
ThingField('subreddit', Subreddit, 'sr_id', 'name'),
#ThingField('reddit',Subreddit,'sr_id','name'),
Field('sr_id'),
Field('url', tokenize=True),
#Field('domain',
# lambda l: UrlParser(l.url).domain_permutations()),
Field('site', lambda l: UrlParser(l.url).domain_permutations()),
#Field('is_self','is_self'),
),
Comment: (
Field('contents', 'body', tokenize=True),
Field(
'boost',
lambda t: int(t._hot * 1000),
# yes, it's a copy of 'hot'
is_number=True,
reverse=True),
ThingField('author', Account, 'author_id', 'name'),
ThingField('subreddit', Subreddit, 'sr_id', 'name'))
}
#ThingField('reddit',Subreddit,'sr_id','name'))}
def POST_options(self, all_langs, **prefs):
u = UrlParser(c.site.path + "prefs")
filter_prefs(prefs, c.user)
if c.errors.errors:
for error in c.errors.errors:
if error[1] == 'stylesheet_override':
u.update_query(error_style_override=error[0])
else:
u.update_query(generic_error=error[0])
return self.redirect(u.unparse())
set_prefs(c.user, prefs)
c.user._commit()
u.update_query(done='true')
if c.cname:
u.put_in_frame()
return self.redirect(u.unparse())