def POST_request_promo(self, srnames):
        if not srnames:
            return

        srnames = srnames.split('+')

        # request multiple ads in case some are hidden by the builder due
        # to the user's hides/preferences
        response = adzerk_request(srnames)

        if not response:
            g.stats.simple_event('adzerk.request.no_promo')
            return

        res_by_campaign = {r.campaign: r for r in response}
        tuples = [promote.PromoTuple(r.link, 1., r.campaign) for r in response]
        builder = CampaignBuilder(tuples, wrap=default_thing_wrapper(),
                                  keep_fn=promote.promo_keep_fn,
                                  num=1,
                                  skip=True)
        listing = LinkListing(builder, nextprev=False).listing()
        if listing.things:
            g.stats.simple_event('adzerk.request.valid_promo')
            w = listing.things[0]
            r = res_by_campaign[w.campaign]

            up = UrlParser(r.imp_pixel)
            up.hostname = "pixel.redditmedia.com"
            w.adserver_imp_pixel = up.unparse()
            w.adserver_click_url = r.click_url
            w.num = ""
            return spaceCompress(w.render())
        else:
            g.stats.simple_event('adzerk.request.skip_promo')
Esempio n. 2
0
    def process_message(msgs, chan):
        """Update get_domain_links(), the Links by domain precomputed query.

        get_domain_links() is a CachedResult which is stored in permacache. To
        update these objects we need to do a read-modify-write which requires
        obtaining a lock. Sharding these updates by domain allows us to run
        multiple consumers (but ideally just one per shard) to avoid lock
        contention.

        """

        from r2.lib.db.queries import add_queries, get_domain_links

        link_names = {msg.body for msg in msgs}
        links = Link._by_fullname(link_names, return_dict=False)
        print 'Processing %r' % (links,)

        links_by_domain = defaultdict(list)
        for link in links:
            parsed = UrlParser(link.url)

            # update the listings for all permutations of the link's domain
            for domain in parsed.domain_permutations():
                links_by_domain[domain].append(link)

        for d, links in links_by_domain.iteritems():
            with g.stats.get_timer("link_vote_processor.domain_queries"):
                add_queries(
                    queries=[
                        get_domain_links(d, sort, "all") for sort in SORTS],
                    insert_items=links,
                )
Esempio n. 3
0
    def GET_framebuster(self, what = None, blah = None):
        """
        renders the contents of the iframe which, on a cname, checks
        if the user is currently logged into reddit.
        
        if this page is hit from the primary domain, redirects to the
        cnamed domain version of the site.  If the user is logged in,
        this cnamed version will drop a boolean session cookie on that
        domain so that subsequent page reloads will be caught in
        middleware and a frame will be inserted around the content.

        If the user is not logged in, previous session cookies will be
        emptied so that subsequent refreshes will not be rendered in
        that pesky frame.
        """
        if not c.site.domain:
            return ""
        elif c.cname:
            return FrameBuster(login = (what == "login")).render()
        else:
            path = "/framebuster/"
            if c.user_is_loggedin:
                path += "login/"
            u = UrlParser(path + str(random.random()))
            u.mk_cname(require_frame = False, subreddit = c.site,
                       port = request.port)
            return self.redirect(u.unparse())
        # the user is not logged in or there is no cname.
        return FrameBuster(login = False).render()
Esempio n. 4
0
    def purge_url(self, url):
        """Purge an image (by url) from imgix.

        Reference: http://www.imgix.com/docs/tutorials/purging-images

        Note that as mentioned in the imgix docs, in order to remove
        an image, this function should be used *after* already
        removing the image from our source, or imgix will just re-fetch
        and replace the image with a new copy even after purging.
        """

        p = UrlParser(url)

        if p.hostname == g.imgix_domain:
            p.hostname = g.imgix_purge_domain
        elif p.hostname == g.imgix_gif_domain:
            p.hostname = g.imgix_gif_purge_domain

        url = p.unparse()

        requests.post(
            "https://api.imgix.com/v2/image/purger",
            auth=(g.secrets["imgix_api_key"], ""),
            data={"url": url},
        )
Esempio n. 5
0
    def format_output_url(cls, url, **kw):
        """
        Helper method used during redirect to ensure that the redirect
        url (assisted by frame busting code or javasctipt) will point
        to the correct domain and not have any extra dangling get
        parameters.  The extensions are also made to match and the
        resulting url is utf8 encoded.

        Node: for development purposes, also checks that the port
        matches the request port
        """
        preserve_extension = kw.pop("preserve_extension", True)
        u = UrlParser(url)

        if u.is_reddit_url():
            # make sure to pass the port along if not 80
            if not kw.has_key('port'):
                kw['port'] = request.port

            # make sure the extensions agree with the current page
            if preserve_extension and c.extension:
                u.set_extension(c.extension)

        # unparse and encode it un utf8
        rv = _force_unicode(u.unparse()).encode('utf8')
        if "\n" in rv or "\r" in rv:
            abort(400)
        return rv
Esempio n. 6
0
  def url_for_title(self, title):
      """Uses the MediaWiki API to get the URL for a wiki page
      with the given title"""
      if title is None:
          return None

      from pylons import g
      cache_key = ('wiki_url_%s' % title).encode('ascii', 'ignore')
      wiki_url = g.cache.get(cache_key)
      if wiki_url is None:
          # http://www.mediawiki.org/wiki/API:Query_-_Properties#info_.2F_in
          api = UrlParser(g.wiki_api_url)
          api.update_query(
              action = 'query',
              titles= title,
              prop = 'info',
              format = 'yaml',
              inprop = 'url'
          )

          try:
              response = urlopen(api.unparse()).read()
              parsed_response = yaml.load(response, Loader=yaml.CLoader)
              page = parsed_response['query']['pages'][0]
          except:
              return None

          wiki_url = page.get('fullurl').strip()

          # Things are created every couple of days so 12 hours seems
          # to be a reasonable cache time
          g.permacache.set(cache_key, wiki_url, time=3600 * 12)

      return wiki_url
Esempio n. 7
0
def _update_redirect_uri(base_redirect_uri, params, as_fragment=False):
    parsed = UrlParser(base_redirect_uri)
    if as_fragment:
        parsed.fragment = urlencode(params)
    else:
        parsed.update_query(**params)
    return parsed.unparse()
Esempio n. 8
0
def allowed_media_preview_url(url):
    p = UrlParser(url)
    if p.has_static_image_extension():
        return True
    for allowed_domain in g.media_preview_domain_whitelist:
        if is_subdomain(p.hostname, allowed_domain):
            return True
    return False
Esempio n. 9
0
    def GET_framebuster(self):
        if c.site.domain and c.user_is_loggedin:
            u = UrlParser(c.site.path + "/frame")
            u.put_in_frame()
            c.cname = True
            return self.redirect(u.unparse())

        return "fail"
Esempio n. 10
0
 def _key_from_url(cls, url):
     if not utils.domain(url) in g.case_sensitive_domains:
         keyurl = _force_utf8(UrlParser.base_url(url.lower()))
     else:
         # Convert only hostname to lowercase
         up = UrlParser(url)
         up.hostname = up.hostname.lower()
         keyurl = _force_utf8(UrlParser.base_url(up.unparse()))
     return keyurl
Esempio n. 11
0
def redirect_to_host(hostname, path=None):
    """Redirect (302) to the specified path and host."""
    if path is None:
        path = request.path

    u = UrlParser(path)
    u.hostname = hostname

    # 307 redirect so request method is retained
    abort(307, location=u.unparse())
    def POST_request_promo(self, srnames, is_mobile_web, platform, loid, is_refresh):
        self.OPTIONS_request_promo()

        if not srnames:
            return

        # backwards compat
        if platform is None:
            platform = "mobile_web" if is_mobile_web else "desktop"

        srnames = srnames.split('+')

        # request multiple ads in case some are hidden by the builder due
        # to the user's hides/preferences
        response = adzerk_request(srnames, self.get_uid(loid),
                                  platform=platform)

        if not response:
            g.stats.simple_event('adzerk.request.no_promo')
            return

        # for adservers, adzerk returns markup so we pass it to the client
        if isinstance(response, AdserverResponse):
            g.stats.simple_event('adzerk.request.adserver')
            return responsive(response.body)

        res_by_campaign = {r.campaign: r for r in response}
        adserver_click_urls = {r.campaign: r.click_url for r in response}
        tuples = [promote.PromoTuple(r.link, 1., r.campaign) for r in response]
        builder = CampaignBuilder(tuples, wrap=default_thing_wrapper(),
                                  keep_fn=promote.promo_keep_fn,
                                  num=1,
                                  skip=True)
        listing = LinkListing(builder, nextprev=False).listing()
        promote.add_trackers(listing.things, c.site, adserver_click_urls=adserver_click_urls)
        promote.update_served(listing.things)
        if listing.things:
            g.stats.simple_event('adzerk.request.valid_promo')
            if is_refresh:
                g.stats.simple_event('adzerk.request.auto_refresh')

            w = listing.things[0]
            r = res_by_campaign[w.campaign]

            up = UrlParser(r.imp_pixel)
            up.hostname = "pixel.redditmedia.com"
            w.adserver_imp_pixel = up.unparse()
            w.adserver_upvote_pixel = r.upvote_pixel
            w.adserver_downvote_pixel = r.downvote_pixel
            w.adserver_click_url = r.click_url
            w.num = ""
            return responsive(w.render(), space_compress=True)
        else:
            g.stats.simple_event('adzerk.request.skip_promo')
Esempio n. 13
0
def make_scraper(url):
    parsed = UrlParser(url)

    if parsed.is_reddit_url():
        if parsed.path.startswith("/live/"):
            try:
                event_id = parsed.path.split("/")[2]
            except IndexError:
                return
            else:
                return _LiveUpdateScraper(event_id)
Esempio n. 14
0
    def test_sign_url(self):
        u = UrlParser('http://examples.imgix.net/frog.jpg?w=100')
        signed_url = self.provider._sign_url(u, 'abcdef')
        self.assertEqual(signed_url.unparse(),
                'http://examples.imgix.net/frog.jpg?w=100&s=cd3bdf071108af73b15c21bdcee5e49c')

        u = UrlParser('http://examples.imgix.net/frog.jpg')
        u.update_query(w=100)
        signed_url = self.provider._sign_url(u, 'abcdef')
        self.assertEqual(signed_url.unparse(),
                'http://examples.imgix.net/frog.jpg?w=100&s=cd3bdf071108af73b15c21bdcee5e49c')
Esempio n. 15
0
def add_to_domain_query_q(link):
    parsed = UrlParser(link.url)
    if not parsed.domain_permutations():
        # no valid domains found
        return

    if g.shard_domain_query_queues:
        domain_shard = hash(parsed.hostname) % 10
        queue_name = "domain_query_%s_q" % domain_shard
    else:
        queue_name = "domain_query_q"
    amqp.add_item(queue_name, link._fullname)
    def run(self, url):
        if not url:
            return None

        u = UrlParser(url)
        # TODO: We should probably set error messages in these cases.
        if not u.is_reddit_url():
            return None

        event_id = re.match(r'/live/(\w+)/?', u.path)
        if not event_id:
            return None

        return VLiveUpdateEvent.run(self, event_id.group(1))
Esempio n. 17
0
def _get_scrape_url(link):
    if not link.is_self:
        sr_name = link.subreddit_slow.name
        if not feature.is_enabled("imgur_gif_conversion", subreddit=sr_name):
            return link.url
        p = UrlParser(link.url)
        # If it's a gif link on imgur, replacing it with gifv should
        # give us the embedly friendly video url
        if is_subdomain(p.hostname, "imgur.com"):
            if p.path_extension().lower() == "gif":
                p.set_extension("gifv")
                return p.unparse()
        return link.url

    urls = extract_urls_from_markdown(link.selftext)
    second_choice = None
    for url in urls:
        p = UrlParser(url)
        if p.is_reddit_url():
            continue
        # If we don't find anything we like better, use the first image.
        if not second_choice:
            second_choice = url
        # This is an optimization for "proof images" in AMAs.
        if is_subdomain(p.netloc, 'imgur.com') or p.has_image_extension():
            return url

    return second_choice
Esempio n. 18
0
    def test_default_prefix(self):
        u = UrlParser('http://i.reddit.com/r/redditdev')
        u.switch_subdomain_by_extension()
        self.assertEquals('http://www.reddit.com/r/redditdev', u.unparse())

        u = UrlParser('http://i.reddit.com/r/redditdev')
        u.switch_subdomain_by_extension('does-not-exist')
        self.assertEquals('http://www.reddit.com/r/redditdev', u.unparse())
Esempio n. 19
0
    def GET_link_id_redirect(self, link):
        if not link:
            abort(404)
        elif not link.subreddit_slow.can_view(c.user):
            # don't disclose the subreddit/title of a post via the redirect url
            abort(403)
        else:
            redirect_url = link.make_permalink_slow(force_domain=True)
        
        query_params = dict(request.GET)
        if query_params:
            url = UrlParser(redirect_url)
            url.update_query(**query_params)
            redirect_url = url.unparse()

        return self.redirect(redirect_url, code=301)
Esempio n. 20
0
File: link.py Progetto: Kluny/reddit
 def by_url_key(cls, url):
     maxlen = 250
     template = 'byurl(%s,%s)'
     keyurl = _force_utf8(UrlParser.base_url(url.lower()))
     hexdigest = md5(keyurl).hexdigest()
     usable_len = maxlen-len(template)-len(hexdigest)
     return template % (hexdigest, keyurl[:usable_len])
    def run(self, url):
        if not url:
            return None

        u = UrlParser(url)
        # TODO: We should probably set error messages in these cases.
        if not u.is_reddit_url():
            return None

        event_id = re.match(r'/live/(\w+)/?', u.path)
        if not event_id:
            return None

        try:
            return models.LiveUpdateEvent._byID(event_id.group(1))
        except tdb_cassandra.NotFound:
            return None
Esempio n. 22
0
    def test_normal_urls(self):
        u = UrlParser('http://www.reddit.com/r/redditdev')
        u.switch_subdomain_by_extension('compact')
        result = u.unparse()
        self.assertEquals('http://i.reddit.com/r/redditdev', result)

        u = UrlParser(result)
        u.switch_subdomain_by_extension('mobile')
        result = u.unparse()
        self.assertEquals('http://m.reddit.com/r/redditdev', result)
Esempio n. 23
0
def add_sr(path, sr_path = True, nocname=False, force_hostname = False):
    """
    Given a path (which may be a full-fledged url or a relative path),
    parses the path and updates it to include the subreddit path
    according to the rules set by its arguments:

     * force_hostname: if True, force the url's hotname to be updated
       even if it is already set in the path, and subject to the
       c.cname/nocname combination.  If false, the path will still
       have its domain updated if no hostname is specified in the url.
    
     * nocname: when updating the hostname, overrides the value of
       c.cname to set the hotname to g.domain.  The default behavior
       is to set the hostname consistent with c.cname.

     * sr_path: if a cname is not used for the domain, updates the
       path to include c.site.path.
    """
    u = UrlParser(path)
    if sr_path and (nocname or not c.cname):
        u.path_add_subreddit(c.site)

    if not u.hostname or force_hostname:
        u.hostname = get_domain(cname = (c.cname and not nocname),
                                subreddit = False)

    if c.render_style == 'mobile':
        u.set_extension('mobile')

    return u.unparse()
Esempio n. 24
0
    def format_output_url(cls, url, **kw):
        """
        Helper method used during redirect to ensure that the redirect
        url (assisted by frame busting code or javasctipt) will point
        to the correct domain and not have any extra dangling get
        parameters.  The extensions are also made to match and the
        resulting url is utf8 encoded.

        Node: for development purposes, also checks that the port
        matches the request port
        """
        u = UrlParser(url)

        if u.is_reddit_url():
            # make sure to pass the port along if not 80
            if not kw.has_key('port'):
                kw['port'] = request.port
    
            # disentagle the cname (for urls that would have
            # cnameframe=1 in them)
            u.mk_cname(**kw)
    
            # make sure the extensions agree with the current page
            if c.extension:
                u.set_extension(c.extension)

        # unparse and encode it un utf8
        return _force_unicode(u.unparse()).encode('utf8')
Esempio n. 25
0
def _oembed_post(thing, **embed_options):
    subreddit = thing.subreddit_slow
    if (not can_view_link_comments(thing) or
            subreddit.type in Subreddit.private_types):
        raise ForbiddenError(errors.POST_NOT_ACCESSIBLE)

    live = ''
    if embed_options.get('live'):
        time = datetime.now(g.tz).isoformat()
        live = 'data-card-created="{}"'.format(time)

    script = ''
    if not embed_options.get('omitscript', False):
        script = format_html(SCRIPT_TEMPLATE,
                             embedly_script=EMBEDLY_SCRIPT,
                             )

    link_url = UrlParser(thing.make_permalink_slow(force_domain=True))
    link_url.update_query(ref='share', ref_source='embed')

    author_name = ""
    if not thing._deleted:
        author = thing.author_slow
        if author._deleted:
            author_name = _("[account deleted]")
        else:
            author_name = author.name

    html = format_html(POST_EMBED_TEMPLATE,
                       live_data_attr=live,
                       link_url=link_url.unparse(),
                       title=websafe(thing.title),
                       subreddit_url=make_url_https(subreddit.path),
                       subreddit_name=subreddit.name,
                       script=script,
                       )

    oembed_response = dict(_OEMBED_BASE,
                           type="rich",
                           title=thing.title,
                           author_name=author_name,
                           html=html,
                           )

    return oembed_response
Esempio n. 26
0
    def test_same_url(self):
        u = UrlParser('http://example.com:8000/a;b?foo=bar&bar=baz#spam')
        u2 = UrlParser('http://example.com:8000/a;b?bar=baz&foo=bar#spam')
        self.assertEquals(u, u2)

        u3 = UrlParser('')
        u3.scheme = 'http'
        u3.hostname = 'example.com'
        u3.port = 8000
        u3.path = '/a'
        u3.params = 'b'
        u3.update_query(foo='bar', bar='baz')
        u3.fragment = 'spam'
        self.assertEquals(u, u3)
Esempio n. 27
0
def _get_scrape_url(link):
    if not link.is_self:
        return link.url

    urls = extract_urls_from_markdown(link.selftext)
    second_choice = None
    for url in urls:
        p = UrlParser(url)
        if p.is_reddit_url():
            continue
        # If we don't find anything we like better, use the first image.
        if not second_choice:
            second_choice = url
        # This is an optimization for "proof images" in AMAs.
        if is_subdomain(p.netloc, 'imgur.com') or p.has_image_extension():
            return url

    return second_choice
Esempio n. 28
0
    def POST_bpoptions(self, all_langs, **prefs):
        u = UrlParser(c.site.path + "prefs")
        bpfilter_prefs(prefs, c.user)
        if c.errors.errors:
            for error in c.errors.errors:
                if error[1] == 'stylesheet_override':
                    u.update_query(error_style_override=error[0])
                else:
                    u.update_query(generic_error=error[0])
            return self.redirect(u.unparse())

        set_prefs(c.user, prefs)
        c.user._commit()
        u.update_query(done='true')
        return self.redirect(u.unparse())
Esempio n. 29
0
File: post.py Progetto: nty-/reddit
    def GET_quarantine(self, dest):
        if not feature.is_enabled('quarantine'):
            return self.abort404()
        sr = UrlParser(dest).get_subreddit()

        # if dest doesn't include a quarantined subreddit,
        # redirect to the homepage or the original destination
        if not sr:
            return self.redirect('/')
        elif isinstance(sr, FakeSubreddit) or sr.is_exposed(c.user):
            return self.redirect(dest)

        return InterstitialPage(
            _("quarantined"),
            content=QuarantineInterstitial(
                sr_name=sr.name,
                logged_in=c.user_is_loggedin,
                email_verified=c.user_is_loggedin and c.user.email_verified,
            ),
        ).render()
Esempio n. 30
0
    def GET_quarantine(self, dest):
        sr = UrlParser(dest).get_subreddit()

        # if dest doesn't include a quarantined subreddit,
        # redirect to the homepage or the original destination
        if not sr:
            return self.redirect('/')
        elif isinstance(sr, FakeSubreddit) or sr.is_exposed(c.user):
            return self.redirect(dest)

        errpage = InterstitialPage(
            _("quarantined"),
            content=QuarantineInterstitial(
                sr_name=sr.name,
                logged_in=c.user_is_loggedin,
                email_verified=c.user_is_loggedin and c.user.email_verified,
            ),
        )
        request.environ['usable_error_content'] = errpage.render()
        self.abort403()
Esempio n. 31
0
    def POST_options(self, all_langs, pref_lang, **kw):
        #temporary. eventually we'll change pref_clickgadget to an
        #integer preference
        kw['pref_clickgadget'] = kw['pref_clickgadget'] and 5 or 0
        if c.user.pref_show_promote is None:
            kw['pref_show_promote'] = None
        elif not kw.get('pref_show_promote'):
            kw['pref_show_promote'] = False

        if not kw.get("pref_over_18") or not c.user.pref_over_18:
            kw['pref_no_profanity'] = True

        if kw.get("pref_no_profanity") or c.user.pref_no_profanity:
            kw['pref_label_nsfw'] = True

        self.set_options(all_langs, pref_lang, **kw)
        u = UrlParser(c.site.path + "prefs")
        u.update_query(done='true')
        if c.cname:
            u.put_in_frame()
        return self.redirect(u.unparse())
Esempio n. 32
0
    def process(thing):
        if thing.deleted:
            return

        thing_cls = thingcls_by_name[thing.thing_type]
        fname = make_fullname(thing_cls, thing.thing_id)
        thing_score = score(thing.ups, thing.downs)
        thing_controversy = controversy(thing.ups, thing.downs)

        for interval, cutoff in cutoff_by_interval.iteritems():
            if thing.timestamp < cutoff:
                continue

            yield ("user/%s/top/%s/%d" %
                   (thing.thing_type, interval, thing.author_id), thing_score,
                   thing.timestamp, fname)
            yield ("user/%s/controversial/%s/%d" %
                   (thing.thing_type, interval, thing.author_id),
                   thing_controversy, thing.timestamp, fname)

            if thing.spam:
                continue

            if thing.thing_type == "link":
                yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
                       thing_score, thing.timestamp, fname)
                yield ("sr/link/controversial/%s/%d" % (interval, thing.sr_id),
                       thing_controversy, thing.timestamp, fname)

                if thing.url:
                    for domain in UrlParser(thing.url).domain_permutations():
                        yield ("domain/link/top/%s/%s" % (interval, domain),
                               thing_score, thing.timestamp, fname)
                        yield ("domain/link/controversial/%s/%s" %
                               (interval, domain), thing_controversy,
                               thing.timestamp, fname)
Esempio n. 33
0
def _update_redirect_uri(base_redirect_uri, params):
    parsed = UrlParser(base_redirect_uri)
    parsed.update_query(**params)
    return parsed.unparse()
Esempio n. 34
0
 def __init__(self, url):
     self.url = url
     # Having the source document's protocol on hand makes it easier to deal
     # with protocol-relative urls we extract from it.
     self.protocol = UrlParser(url).scheme
Esempio n. 35
0
def add_sr(path,
           sr_path=True,
           nocname=False,
           force_hostname=False,
           retain_extension=True):
    """
    Given a path (which may be a full-fledged url or a relative path),
    parses the path and updates it to include the subreddit path
    according to the rules set by its arguments:

     * force_hostname: if True, force the url's hostname to be updated
       even if it is already set in the path, and subject to the
       c.cname/nocname combination.  If false, the path will still
       have its domain updated if no hostname is specified in the url.

     * nocname: when updating the hostname, overrides the value of
       c.cname to set the hostname to g.domain.  The default behavior
       is to set the hostname consistent with c.cname.

     * sr_path: if a cname is not used for the domain, updates the
       path to include c.site.path.

    For caching purposes: note that this function uses:
      c.cname, c.render_style, c.site.name
    """
    # don't do anything if it is just an anchor
    if path.startswith(('#', 'javascript:')):
        return path

    u = UrlParser(path)
    if sr_path and (nocname or not c.cname):
        u.path_add_subreddit(c.site)

    if not u.hostname or force_hostname:
        if c.secure:
            u.hostname = request.host
        else:
            u.hostname = get_domain(cname=(c.cname and not nocname),
                                    subreddit=False)

    if c.secure:
        u.scheme = "https"

    if retain_extension:
        if c.render_style == 'mobile':
            u.set_extension('mobile')

        elif c.render_style == 'compact':
            u.set_extension('compact')

    return u.unparse()
Esempio n. 36
0
 def test_empty_extension(self):
     u = UrlParser('http://example.com/a.')
     self.assertEquals('', u.path_extension())
Esempio n. 37
0
    def test_same_url(self):
        u = UrlParser('http://example.com:8000/a;b?foo=bar&bar=baz#spam')
        u2 = UrlParser('http://example.com:8000/a;b?bar=baz&foo=bar#spam')
        self.assertEquals(u, u2)

        u3 = UrlParser('')
        u3.scheme = 'http'
        u3.hostname = 'example.com'
        u3.port = 8000
        u3.path = '/a'
        u3.params = 'b'
        u3.update_query(foo='bar', bar='baz')
        u3.fragment = 'spam'
        self.assertEquals(u, u3)
Esempio n. 38
0
def make_feedurl(user, path, ext="rss"):
    u = UrlParser(path)
    u.update_query(user=user.name, feed=make_feedhash(user, path))
    u.set_extension(ext)
    return u.unparse()
Esempio n. 39
0
 def test_different_domains(self):
     u = UrlParser('http://example.com')
     u2 = UrlParser('http://example.org')
     self.assertNotEquals(u, u2)
Esempio n. 40
0
 def test_unicode_query_params(self):
     u = UrlParser(u'http://example.com/?page=unicode:(')
     u2 = UrlParser('http://example.com/')
     u2.update_query(page=u'unicode:(')
     self.assertEquals(u, u2)
Esempio n. 41
0
 def test_different_fragments(self):
     u = UrlParser('http://example.com/')
     u2 = UrlParser('http://example.com/#foo')
     u3 = UrlParser('http://example.com/#bar')
     self.assertNotEquals(u, u2)
     self.assertNotEquals(u2, u3)
Esempio n. 42
0
 def test_different_protocols(self):
     u = UrlParser('http://example.com')
     u2 = UrlParser('https://example.com')
     self.assertNotEquals(u, u2)
Esempio n. 43
0
 def test_different_paths(self):
     u = UrlParser('http://example.com')
     u2 = UrlParser('http://example.com/a')
     u3 = UrlParser('http://example.com/b')
     self.assertNotEquals(u, u2)
     self.assertNotEquals(u2, u3)
Esempio n. 44
0
 def _is_safe_reddit_url(self, url, subreddit=None):
     web_safe = UrlParser(url).is_web_safe_url()
     return web_safe and UrlParser(url).is_reddit_url(subreddit)
Esempio n. 45
0
 def _key_from_url(cls, url):
     keyurl = _force_utf8(UrlParser.base_url(url.lower()))
     return keyurl
Esempio n. 46
0
 def test_nested_file(self):
     u = UrlParser('http://example.com/foo/a.jpg')
     self.assertEquals('jpg', u.path_extension())
def add_sr(path,
           sr_path=True,
           nocname=False,
           force_hostname=False,
           retain_extension=True,
           force_https=False,
           force_extension=None):
    """
    Given a path (which may be a full-fledged url or a relative path),
    parses the path and updates it to include the subreddit path
    according to the rules set by its arguments:

     * sr_path: if a cname is not used for the domain, updates the
       path to include c.site.path.

     * nocname: deprecated.

     * force_hostname: if True, force the url's hostname to be updated
       even if it is already set in the path. If false, the path will still
       have its domain updated if no hostname is specified in the url.

     * retain_extension: if True, sets the extention according to
       c.render_style.

     * force_https: force the URL scheme to https

    For caching purposes: note that this function uses:
      c.render_style, c.site.name

    """
    # don't do anything if it is just an anchor
    if path.startswith(('#', 'javascript:')):
        return path

    u = UrlParser(path)
    if sr_path:
        u.path_add_subreddit(c.site)

    if not u.hostname or force_hostname:
        u.hostname = get_domain(subreddit=False)

    if (c.secure and u.is_reddit_url()) or force_https:
        u.scheme = "https"

    if force_extension is not None:
        u.set_extension(force_extension)
    elif retain_extension:
        if c.render_style == 'mobile':
            u.set_extension('mobile')

        elif c.render_style == 'compact':
            u.set_extension('compact')

    return u.unparse()
Esempio n. 48
0
 def add_ext_to_link(link):
     url = UrlParser(link.get('href'))
     if url.is_reddit_url():
         link['href'] = add_sr(link.get('href'), sr_path=False)
def update_query(base_url, **kw):
    parsed = UrlParser(base_url)
    parsed.update_query(**kw)
    return parsed.unparse()
Esempio n. 50
0
 def test_only_extension(self):
     u = UrlParser('http://example.com/.bashrc')
     self.assertEquals('bashrc', u.path_extension())
Esempio n. 51
0
 def test_two_extensions(self):
     u = UrlParser('http://example.com/a.jpg.exe')
     self.assertEquals('exe', u.path_extension())
Esempio n. 52
0
 def test_different_ports(self):
     u = UrlParser('http://example.com')
     u2 = UrlParser('http://example.com:8000')
     u3 = UrlParser('http://example.com:8008')
     self.assertNotEquals(u, u2)
     self.assertNotEquals(u2, u3)
Esempio n. 53
0
 def test_integer_query_params(self):
     u = UrlParser('http://example.com/?page=1234')
     u2 = UrlParser('http://example.com/')
     u2.update_query(page=1234)
     self.assertEquals(u, u2)
Esempio n. 54
0
 def test_different_params(self):
     u = UrlParser('http://example.com/')
     u2 = UrlParser('http://example.com/;foo')
     u3 = UrlParser('http://example.com/;bar')
     self.assertNotEquals(u, u2)
     self.assertNotEquals(u2, u3)
Esempio n. 55
0
    def resize_image(self,
                     image,
                     width=None,
                     censor_nsfw=False,
                     max_ratio=None):
        url = UrlParser(image['url'])
        url.hostname = g.imgix_domain
        # Let's encourage HTTPS; it's cool, works just fine on HTTP pages, and
        # will prevent insecure content warnings on HTTPS pages.
        url.scheme = 'https'

        if max_ratio:
            url.update_query(fit='crop')
            # http://www.imgix.com/docs/reference/size#param-crop
            url.update_query(crop='faces,entropy')
            url.update_query(arh=max_ratio)

        if width:
            if width > image['width']:
                raise NotLargeEnough()
            # http://www.imgix.com/docs/reference/size#param-w
            url.update_query(w=width)
        if censor_nsfw:
            # Do an initial blur to make sure we're getting rid of icky
            # details.
            #
            # http://www.imgix.com/docs/reference/stylize#param-blur
            url.update_query(blur=600)
            # And then add pixellation to help the image compress well.
            #
            # http://www.imgix.com/docs/reference/stylize#param-px
            url.update_query(px=32)
        if g.imgix_signing:
            url = self._sign_url(url, g.secrets['imgix_signing_token'])
        return url.unparse()
Esempio n. 56
0
    def POST_edit_promo(self, form, jquery, username, l, title, url, selftext,
                        kind, disable_comments, sendreplies, media_url,
                        media_autoplay, media_override, gifts_embed_url,
                        media_url_type, domain_override, is_managed):

        should_ratelimit = False
        if not c.user_is_sponsor:
            should_ratelimit = True

        if not should_ratelimit:
            c.errors.remove((errors.RATELIMIT, 'ratelimit'))

        # check for user override
        if not l and c.user_is_sponsor and username:
            try:
                user = Account._by_name(username)
            except NotFound:
                c.errors.add(errors.USER_DOESNT_EXIST, field="username")
                form.set_error(errors.USER_DOESNT_EXIST, "username")
                return

            if not user.email:
                c.errors.add(errors.NO_EMAIL_FOR_USER, field="username")
                form.set_error(errors.NO_EMAIL_FOR_USER, "username")
                return

            if not user.email_verified:
                c.errors.add(errors.NO_VERIFIED_EMAIL, field="username")
                form.set_error(errors.NO_VERIFIED_EMAIL, "username")
                return
        else:
            user = c.user

        # check for shame banned domains
        if form.has_errors("url", errors.DOMAIN_BANNED):
            g.stats.simple_event('spam.shame.link')
            return

        # demangle URL in canonical way
        if url:
            if isinstance(url, (unicode, str)):
                form.set_inputs(url=url)
            elif isinstance(url, tuple) or isinstance(url[0], Link):
                # there's already one or more links with this URL, but
                # we're allowing mutliple submissions, so we really just
                # want the URL
                url = url[0].url

        if kind == 'link':
            if form.has_errors('url', errors.NO_URL, errors.BAD_URL):
                return

        # users can change the disable_comments on promoted links
        if ((not l or not promote.is_promoted(l))
                and (form.has_errors('title', errors.NO_TEXT, errors.TOO_LONG)
                     or jquery.has_errors('ratelimit', errors.RATELIMIT))):
            return

        if kind == 'self' and form.has_errors('text', errors.TOO_LONG):
            return

        if not l:
            # creating a new promoted link
            l = promote.new_promotion(title, url if kind == 'link' else 'self',
                                      selftext if kind == 'self' else '', user,
                                      request.ip)
            l.domain_override = domain_override or None
            if c.user_is_sponsor:
                l.managed_promo = is_managed
            l._commit()
            form.redirect(promote.promo_edit_url(l))

        elif not promote.is_promo(l):
            return

        # changing link type is not allowed
        if ((l.is_self and kind == 'link')
                or (not l.is_self and kind == 'self')):
            c.errors.add(errors.NO_CHANGE_KIND, field="kind")
            form.set_error(errors.NO_CHANGE_KIND, "kind")
            return

        changed = False
        # live items can only be changed by a sponsor, and also
        # pay the cost of de-approving the link
        if not promote.is_promoted(l) or c.user_is_sponsor:
            if title and title != l.title:
                l.title = title
                changed = not c.user_is_sponsor

            if kind == 'link' and url and url != l.url:
                l.url = url
                changed = not c.user_is_sponsor

        # only trips if the title and url are changed by a non-sponsor
        if changed:
            promote.unapprove_promotion(l)

        # selftext can be changed at any time
        if kind == 'self':
            l.selftext = selftext

        # comment disabling and sendreplies is free to be changed any time.
        l.disable_comments = disable_comments
        l.sendreplies = sendreplies

        if c.user_is_sponsor:
            if (form.has_errors("media_url", errors.BAD_URL)
                    or form.has_errors("gifts_embed_url", errors.BAD_URL)):
                return

        scraper_embed = media_url_type == "scrape"
        media_url = media_url or None
        gifts_embed_url = gifts_embed_url or None

        if c.user_is_sponsor and scraper_embed and media_url != l.media_url:
            if media_url:
                media = _scrape_media(media_url,
                                      autoplay=media_autoplay,
                                      save_thumbnail=False,
                                      use_cache=True)

                if media:
                    l.set_media_object(media.media_object)
                    l.set_secure_media_object(media.secure_media_object)
                    l.media_url = media_url
                    l.gifts_embed_url = None
                    l.media_autoplay = media_autoplay
                else:
                    c.errors.add(errors.SCRAPER_ERROR, field="media_url")
                    form.set_error(errors.SCRAPER_ERROR, "media_url")
                    return
            else:
                l.set_media_object(None)
                l.set_secure_media_object(None)
                l.media_url = None
                l.gifts_embed_url = None
                l.media_autoplay = False

        if (c.user_is_sponsor and not scraper_embed
                and gifts_embed_url != l.gifts_embed_url):
            if gifts_embed_url:
                parsed = UrlParser(gifts_embed_url)
                if not is_subdomain(parsed.hostname, "redditgifts.com"):
                    c.errors.add(errors.BAD_URL, field="gifts_embed_url")
                    form.set_error(errors.BAD_URL, "gifts_embed_url")
                    return

                iframe = """
                    <iframe class="redditgifts-embed"
                            src="%(embed_url)s"
                            width="710" height="500" scrolling="no"
                            frameborder="0" allowfullscreen>
                    </iframe>
                """ % {
                    'embed_url': websafe(gifts_embed_url)
                }
                media_object = {
                    'oembed': {
                        'description': 'redditgifts embed',
                        'height': 500,
                        'html': iframe,
                        'provider_name': 'redditgifts',
                        'provider_url': 'http://www.redditgifts.com/',
                        'title': 'redditgifts secret santa 2014',
                        'type': 'rich',
                        'width': 710
                    },
                    'type': 'redditgifts'
                }
                l.set_media_object(media_object)
                l.set_secure_media_object(media_object)
                l.media_url = None
                l.gifts_embed_url = gifts_embed_url
                l.media_autoplay = False
            else:
                l.set_media_object(None)
                l.set_secure_media_object(None)
                l.media_url = None
                l.gifts_embed_url = None
                l.media_autoplay = False

        if c.user_is_sponsor:
            l.media_override = media_override
            l.domain_override = domain_override or None
            l.managed_promo = is_managed

        l._commit()
        form.redirect(promote.promo_edit_url(l))
Esempio n. 57
0
 def test_different_objects(self):
     u = UrlParser('http://example.com')
     self.assertNotEquals(u, None)
Esempio n. 58
0
 def test_different_queries(self):
     u = UrlParser('http://example.com/')
     u2 = UrlParser('http://example.com/?foo')
     u3 = UrlParser('http://example.com/?foo=bar')
     self.assertNotEquals(u, u2)
     self.assertNotEquals(u2, u3)
Esempio n. 59
0
        Field('contents', 'title', tokenize=True),
        Field(
            'boost',
            lambda t: int(t._hot * 1000),
            # yes, it's a copy of 'hot'
            is_number=True,
            reverse=True),
        Field('author_id'),
        ThingField('author', Account, 'author_id', 'name'),
        ThingField('subreddit', Subreddit, 'sr_id', 'name'),
        #ThingField('reddit',Subreddit,'sr_id','name'),
        Field('sr_id'),
        Field('url', tokenize=True),
        #Field('domain',
        #      lambda l: UrlParser(l.url).domain_permutations()),
        Field('site', lambda l: UrlParser(l.url).domain_permutations()),
        #Field('is_self','is_self'),
    ),
    Comment: (
        Field('contents', 'body', tokenize=True),
        Field(
            'boost',
            lambda t: int(t._hot * 1000),
            # yes, it's a copy of 'hot'
            is_number=True,
            reverse=True),
        ThingField('author', Account, 'author_id', 'name'),
        ThingField('subreddit', Subreddit, 'sr_id', 'name'))
}
#ThingField('reddit',Subreddit,'sr_id','name'))}
Esempio n. 60
0
    def POST_options(self, all_langs, **prefs):
        u = UrlParser(c.site.path + "prefs")

        filter_prefs(prefs, c.user)
        if c.errors.errors:
            for error in c.errors.errors:
                if error[1] == 'stylesheet_override':
                    u.update_query(error_style_override=error[0])
                else:
                    u.update_query(generic_error=error[0])
            return self.redirect(u.unparse())

        set_prefs(c.user, prefs)
        c.user._commit()
        u.update_query(done='true')
        if c.cname:
            u.put_in_frame()
        return self.redirect(u.unparse())