Exemple #1
0
    def log_hit(self):
        # Make sure we get plain strings, not unicode
        #self.request.charset = None 
        cur = self.request.str_GET.get('cur', None)
        ref = self.request.str_GET.get('ref', None)
        title = self.request.str_GET.get('title', None)

        if not cur:
            cur = self.request.headers.get('referer', None)
        
        if cur is not None:
            cur = get_unicode(unquote_plus(cur))
        if ref is not None:
            ref = get_unicode(unquote_plus(ref))
        if title is not None:
            title = get_unicode(unquote_plus(title)).strip()

        response = Response()
        response.headers['Content-Type'] = "image/gif"
        response.headers['Expires'] = "Sat, 26 Jul 1997 05:00:00 GMT"
        response.headers['Cache-Control'] = "no-cache, must-revalidate"
        response.body = self._image

        if not cur:
            return response

        visitor_ip = self.request.headers.get('X-Forwarded-For', None)
        if visitor_ip is None:
            visitor_ip = self.request.remote_addr

        hit = Hit(self._conf, cur, referrer=ref, title=title,
            visitor_ip=visitor_ip)
        if hit.is_whitelisted() and not hit.is_blacklisted():
            self._storage.add_hit(hit)
        return response
Exemple #2
0
    def test_timestamp(self):
        """Test the timestamp as (signed) int, string and None"""
        tests = []
        # Zero
        tests.append({'timestamp': 0, 'expect': 0})
        # Small number
        tests.append({'timestamp': 1000, 'expect': 1000})
        # Negative number
        tests.append({'timestamp': -1000, 'expect': -1000})
        # Regular size
        tests.append({'timestamp': 9879824154, 'expect': 9879824154 })
        # Big number
        tests.append({'timestamp': 98798241549879824154,
                      'expect': 98798241549879824154})
        # As a string
        tests.append({'timestamp': '-1000', 'expect': -1000})
        # None (fallback to current timestamp)
        tests.append({'timestamp': None, 'expect': self.now})

        for test in tests:
            self.hit = Hit(self._conf, u'http://abc.nl',
                timestamp=test['timestamp'])
            self.assertEqual(self.hit.timestamp(), test['expect'])
Exemple #3
0
 def test_whitelist_single_matching(self):
     """Test path whitelisting - single, matching"""
     testconf = deepcopy(self._conf)
     testconf['whitelist_lvl1'] = 'path'
     hit = Hit(testconf, u'http://abc.nl/path/to/page')
     self.assertEqual(hit.is_whitelisted(), True)
Exemple #4
0
 def test_referrer_with_special_chars(self):
     """Test a referrer with special characters in it"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://xyz.nl/page?a=éáëöóñùüẽ')
     self.assertEqual(self.hit.referrer(),
             u'http://xyz.nl/page?a=éáëöóñùüẽ')
Exemple #5
0
 def test_url_with_ending_slash(self):
     """Test a url with an ending slash"""
     self.hit = Hit(self._conf, u'http://www.mysite.com/page')
     self.assertEqual(self.hit.url(), u'http://www.mysite.com/page')
     self.hit = Hit(self._conf, u'http://www.mysite.com/page/')
     self.assertEqual(self.hit.url(), u'http://www.mysite.com/page')
Exemple #6
0
 def test_path_with_cyrillic_chars(self):
     """Test a path with cyrillic (russian) characters in it"""
     self.hit = Hit(self._conf, u'http://mysite.com/page?a=русское альфа')
     self.assertEqual(self.hit.path(), u'/page?a=русское альфа')
Exemple #7
0
class TestHit(TestBase):

    def test_url_root(self):
        """Test the root url"""
        self.hit = Hit(self._conf, u'http://mysite.com')
        self.assertEqual(self.hit.url(), u'http://mysite.com')
        self.hit = Hit(self._conf, u'http://mysite.com/')
        self.assertEqual(self.hit.url(), u'http://mysite.com')

    def test_url_with_subdomain(self):
        """Test a url with a subdomain"""
        self.hit = Hit(self._conf, u'http://abc.mysite.com')
        self.assertEqual(self.hit.url(), u'http://abc.mysite.com')

    def test_url_with_ending_slash(self):
        """Test a url with an ending slash"""
        self.hit = Hit(self._conf, u'http://www.mysite.com/page')
        self.assertEqual(self.hit.url(), u'http://www.mysite.com/page')
        self.hit = Hit(self._conf, u'http://www.mysite.com/page/')
        self.assertEqual(self.hit.url(), u'http://www.mysite.com/page')

    def test_url_with_file_extension(self):
        """Test a url with a file extension"""
        self.hit = Hit(self._conf, u'http://mysite.com/page.html')
        self.assertEqual(self.hit.url(), u'http://mysite.com/page.html')

    def test_url_with_query_parameters(self):
        """Test a url with query string parameters and an anchor"""
        self.hit = Hit(self._conf, u'http://mysite.com/page?a=x&b=1#top')
        self.assertEqual(self.hit.url(), u'http://mysite.com/page?a=x&b=1#top')

    def test_url_with_cyrillic_chars(self):
        """Test a url with cyrillic (russian) characters in it"""
        self.hit = Hit(self._conf, u'http://mysite.com/page?a=русское альфа')
        self.assertEqual(self.hit.url(),
            u'http://mysite.com/page?a=русское альфа')

    def test_url_utf8(self):
        """Test a url that is provided as utf-8 encoded string"""
        self.hit = Hit(self._conf,
                u'http://mysite.com/café'.encode('utf8'))
        self.assertEqual(self.hit.url(), 
                u'http://mysite.com/café'.encode('utf8'))

    def test_url_latin1(self):
        """Test a url that is provided as latin-1 encoded string"""
        self.hit = Hit(self._conf,
                u'http://mysite.com/café'.encode('latin1'))
        self.assertEqual(self.hit.url(), 
                u'http://mysite.com/café'.encode('latin1'))
    
    def test_path_with_subdomain(self):
        """Test a path with a subdomain"""
        self.hit = Hit(self._conf, u'http://www.mysite.com/page')
        self.assertEqual(self.hit.path(), u'/page')

    def test_path_with_file_extension(self):
        """Test a path with a file extension"""
        self.hit = Hit(self._conf, u'http://mysite.com/page.html')
        self.assertEqual(self.hit.path(), u'/page.html')

    def test_path_with_query_parameters(self):
        """Test a path with query string parameters and an anchor"""
        self.hit = Hit(self._conf, u'http://mysite.com/page?a=x&b=1#top')
        self.assertEqual(self.hit.path(), u'/page?a=x&b=1')

    def test_path_with_cyrillic_chars(self):
        """Test a path with cyrillic (russian) characters in it"""
        self.hit = Hit(self._conf, u'http://mysite.com/page?a=русское альфа')
        self.assertEqual(self.hit.path(), u'/page?a=русское альфа')

    def test_path_utf8(self):
        """Test a path that is provided as utf-8 encoded string"""
        self.hit = Hit(self._conf,
                u'http://mysite.com/café'.encode('utf8'))
        self.assertEqual(self.hit.path(), u'/café'.encode('utf8'))

    def test_path_latin1(self):
        """Test a path that is provided as latin-1 encoded string"""
        self.hit = Hit(self._conf,
                u'http://mysite.com/café'.encode('latin1'))
        self.assertEqual(self.hit.path(), u'/café'.encode('latin1'))
    
    def test_title_basic(self):
        """Test a title with basic characters"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                title='ABC.nl - Some random page')
        self.assertEqual(self.hit.title(), 'ABC.nl - Some random page')

    def test_title_specialchars(self):
        """Test a title with special characters"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                title='<>@!^#$%&*¶«{(?)}»~€⁂⁀®“『₳”"')
        self.assertEqual(self.hit.title(),
                '<>@!^#$%&*¶«{(?)}»~€⁂⁀®“『₳”"')

    def test_title_cyrillic(self):
        """Test a title with cyrillic characters"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                title='До последней, это до которой конкретно?')
        self.assertEqual(self.hit.title(),
                'До последней, это до которой конкретно?')

    def test_referrer_with_subdomain(self):
        """Test a referrer with a subdomain"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://abc.xyz.nl/some/where')
        self.assertEqual(self.hit.referrer(), u'http://abc.xyz.nl/some/where')

    def test_referrer_with_file_extension(self):
        """Test a referrer with a file extension"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://xyz.nl/page.html')
        self.assertEqual(self.hit.referrer(), u'http://xyz.nl/page.html')

    def test_referrer_with_query_parameters(self):
        """Test a referrer with query string parameters and an anchor"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://xyz.nl/page?a=x&b=1#top')
        self.assertEqual(self.hit.referrer(),
                u'http://xyz.nl/page?a=x&b=1#top')

    def test_referrer_with_special_chars(self):
        """Test a referrer with special characters in it"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://xyz.nl/page?a=éáëöóñùüẽ')
        self.assertEqual(self.hit.referrer(),
                u'http://xyz.nl/page?a=éáëöóñùüẽ')

    def test_referrer_with_cyrillic_chars(self):
        """Test a referrer with cyrillic (russian) characters in it"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://xyz.nl/page?a=русское альфа')
        self.assertEqual(self.hit.referrer(),
                u'http://xyz.nl/page?a=русское альфа')
    
    def test_referrer_utf8(self):
        """Test a referrer with utf-8 encoding"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://xyz.nl/page?a=café'.encode('utf-8'))
        self.assertEqual(self.hit.referrer(),
                u'http://xyz.nl/page?a=café'.encode('utf-8'))
    
    def test_referrer_latin1(self):
        """Test a referrer with latin-1 encoding"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://xyz.nl/page?a=café'.encode('latin-1'))
        self.assertEqual(self.hit.referrer(),
                u'http://xyz.nl/page?a=café'.encode('latin-1'))

    def test_source_direct(self):
        """Test source - direct (no referrer)"""
        self.hit = Hit(self._conf, u'http://abc.nl/page')
        self.assertEqual(self.hit.source(), u'direct')

    def test_source_internal(self):
        """Test source - internal"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://abc.nl/page2')
        self.assertEqual(self.hit.source(), u'internal')

    def test_source_external(self):
        """Test source - external"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://www.xyz.nl/page')
        self.assertEqual(self.hit.source(), u'external: www.xyz.nl')
    
    def test_source_searches(self):
        """Test source - searches (basic)"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://google.nl?q=abc')
        self.assertEqual(self.hit.source(), u'searches - google.nl: abc')

    def test_source_searches_special_chars(self):
        """Test source - searches (special characters)"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://google.nl?q=éáëöóñùüẽ')
        self.assertEqual(self.hit.source(), u'searches - google.nl: éáëöóñùüẽ')

    def test_source_searches_cyrillic_chars(self):
        """Test source - searches (cyrillic characters)"""
        self.hit = Hit(self._conf, u'http://abc.nl/page',
                referrer=u'http://google.ru?q=русское альфа')
        self.assertEqual(self.hit.source(),
                u'searches - google.ru: русское альфа')

    def test_timestamp(self):
        """Test the timestamp as (signed) int, string and None"""
        tests = []
        # Zero
        tests.append({'timestamp': 0, 'expect': 0})
        # Small number
        tests.append({'timestamp': 1000, 'expect': 1000})
        # Negative number
        tests.append({'timestamp': -1000, 'expect': -1000})
        # Regular size
        tests.append({'timestamp': 9879824154, 'expect': 9879824154 })
        # Big number
        tests.append({'timestamp': 98798241549879824154,
                      'expect': 98798241549879824154})
        # As a string
        tests.append({'timestamp': '-1000', 'expect': -1000})
        # None (fallback to current timestamp)
        tests.append({'timestamp': None, 'expect': self.now})

        for test in tests:
            self.hit = Hit(self._conf, u'http://abc.nl',
                timestamp=test['timestamp'])
            self.assertEqual(self.hit.timestamp(), test['expect'])

    def test_whitelist_empty(self):
        """Test path whitelisting - empty"""
        testconf = deepcopy(self._conf)
        testconf['whitelist_lvl1'] = ''
        hit = Hit(testconf, u'http://abc.nl/path/to/page')
        self.assertEqual(hit.is_whitelisted(), True)

    def test_whitelist_single_matching(self):
        """Test path whitelisting - single, matching"""
        testconf = deepcopy(self._conf)
        testconf['whitelist_lvl1'] = 'path'
        hit = Hit(testconf, u'http://abc.nl/path/to/page')
        self.assertEqual(hit.is_whitelisted(), True)

    def test_whitelist_single_nonmatching(self):
        """Test path whitelisting - single, non-matchingempty"""
        testconf = deepcopy(self._conf)
        testconf['whitelist_lvl1'] = 'anotherpath'
        hit = Hit(testconf, u'http://abc.nl/path/to/page')
        self.assertEqual(hit.is_whitelisted(), False)

    def test_whitelist_multi_matching(self):
        """Test path whitelisting - multi, matching"""
        testconf = deepcopy(self._conf)
        testconf['whitelist_lvl1'] = 'one,path,three'
        hit = Hit(testconf, u'http://abc.nl/path/to/page')
        self.assertEqual(hit.is_whitelisted(), True)

    def test_whitelist_multi_nonmatching(self):
        """Test path whitelisting - multi, non-matching"""
        testconf = deepcopy(self._conf)
        testconf['whitelist_lvl1'] = 'one,two,three'
        hit = Hit(testconf, u'http://abc.nl/path/to/page')
        self.assertEqual(hit.is_whitelisted(), False)

    def test_whitelist_multi_spaced(self):
        """Test path whitelisting - multi, with spaces"""
        testconf = deepcopy(self._conf)
        testconf['whitelist_lvl1'] = 'one, path , three'
        hit = Hit(testconf, u'http://abc.nl/path/to/page')
        self.assertEqual(hit.is_whitelisted(), True)
Exemple #8
0
 def test_whitelist_multi_spaced(self):
     """Test path whitelisting - multi, with spaces"""
     testconf = deepcopy(self._conf)
     testconf['whitelist_lvl1'] = 'one, path , three'
     hit = Hit(testconf, u'http://abc.nl/path/to/page')
     self.assertEqual(hit.is_whitelisted(), True)
Exemple #9
0
 def test_path_with_file_extension(self):
     """Test a path with a file extension"""
     self.hit = Hit(self._conf, u'http://mysite.com/page.html')
     self.assertEqual(self.hit.path(), u'/page.html')
Exemple #10
0
 def test_source_external(self):
     """Test source - external"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://www.xyz.nl/page')
     self.assertEqual(self.hit.source(), u'external: www.xyz.nl')
Exemple #11
0
 def test_url_with_subdomain(self):
     """Test a url with a subdomain"""
     self.hit = Hit(self._conf, u'http://abc.mysite.com')
     self.assertEqual(self.hit.url(), u'http://abc.mysite.com')
Exemple #12
0
 def test_source_direct(self):
     """Test source - direct (no referrer)"""
     self.hit = Hit(self._conf, u'http://abc.nl/page')
     self.assertEqual(self.hit.source(), u'direct')
Exemple #13
0
 def test_source_internal(self):
     """Test source - internal"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://abc.nl/page2')
     self.assertEqual(self.hit.source(), u'internal')
Exemple #14
0
 def test_referrer_latin1(self):
     """Test a referrer with latin-1 encoding"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://xyz.nl/page?a=café'.encode('latin-1'))
     self.assertEqual(self.hit.referrer(),
             u'http://xyz.nl/page?a=café'.encode('latin-1'))
Exemple #15
0
 def test_referrer_utf8(self):
     """Test a referrer with utf-8 encoding"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://xyz.nl/page?a=café'.encode('utf-8'))
     self.assertEqual(self.hit.referrer(),
             u'http://xyz.nl/page?a=café'.encode('utf-8'))
Exemple #16
0
 def test_referrer_with_cyrillic_chars(self):
     """Test a referrer with cyrillic (russian) characters in it"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://xyz.nl/page?a=русское альфа')
     self.assertEqual(self.hit.referrer(),
             u'http://xyz.nl/page?a=русское альфа')
Exemple #17
0
 def test_whitelist_single_nonmatching(self):
     """Test path whitelisting - single, non-matchingempty"""
     testconf = deepcopy(self._conf)
     testconf['whitelist_lvl1'] = 'anotherpath'
     hit = Hit(testconf, u'http://abc.nl/path/to/page')
     self.assertEqual(hit.is_whitelisted(), False)
Exemple #18
0
 def test_source_searches(self):
     """Test source - searches (basic)"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://google.nl?q=abc')
     self.assertEqual(self.hit.source(), u'searches - google.nl: abc')
Exemple #19
0
 def test_whitelist_multi_nonmatching(self):
     """Test path whitelisting - multi, non-matching"""
     testconf = deepcopy(self._conf)
     testconf['whitelist_lvl1'] = 'one,two,three'
     hit = Hit(testconf, u'http://abc.nl/path/to/page')
     self.assertEqual(hit.is_whitelisted(), False)
Exemple #20
0
 def test_source_searches_special_chars(self):
     """Test source - searches (special characters)"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://google.nl?q=éáëöóñùüẽ')
     self.assertEqual(self.hit.source(), u'searches - google.nl: éáëöóñùüẽ')
Exemple #21
0
 def test_path_with_subdomain(self):
     """Test a path with a subdomain"""
     self.hit = Hit(self._conf, u'http://www.mysite.com/page')
     self.assertEqual(self.hit.path(), u'/page')
Exemple #22
0
 def test_source_searches_cyrillic_chars(self):
     """Test source - searches (cyrillic characters)"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://google.ru?q=русское альфа')
     self.assertEqual(self.hit.source(),
             u'searches - google.ru: русское альфа')
Exemple #23
0
 def test_path_with_query_parameters(self):
     """Test a path with query string parameters and an anchor"""
     self.hit = Hit(self._conf, u'http://mysite.com/page?a=x&b=1#top')
     self.assertEqual(self.hit.path(), u'/page?a=x&b=1')
Exemple #24
0
 def test_title_basic(self):
     """Test a title with basic characters"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             title='ABC.nl - Some random page')
     self.assertEqual(self.hit.title(), 'ABC.nl - Some random page')
Exemple #25
0
 def test_path_utf8(self):
     """Test a path that is provided as utf-8 encoded string"""
     self.hit = Hit(self._conf,
             u'http://mysite.com/café'.encode('utf8'))
     self.assertEqual(self.hit.path(), u'/café'.encode('utf8'))
Exemple #26
0
 def test_referrer_with_file_extension(self):
     """Test a referrer with a file extension"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://xyz.nl/page.html')
     self.assertEqual(self.hit.referrer(), u'http://xyz.nl/page.html')
Exemple #27
0
 def test_path_latin1(self):
     """Test a path that is provided as latin-1 encoded string"""
     self.hit = Hit(self._conf,
             u'http://mysite.com/café'.encode('latin1'))
     self.assertEqual(self.hit.path(), u'/café'.encode('latin1'))
Exemple #28
0
 def test_referrer_with_query_parameters(self):
     """Test a referrer with query string parameters and an anchor"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             referrer=u'http://xyz.nl/page?a=x&b=1#top')
     self.assertEqual(self.hit.referrer(),
             u'http://xyz.nl/page?a=x&b=1#top')
Exemple #29
0
 def test_title_specialchars(self):
     """Test a title with special characters"""
     self.hit = Hit(self._conf, u'http://abc.nl/page',
             title='<>@!^#$%&*¶«{(?)}»~€⁂⁀®“『₳”"')
     self.assertEqual(self.hit.title(),
             '<>@!^#$%&*¶«{(?)}»~€⁂⁀®“『₳”"')
Exemple #30
0
 def test_whitelist_empty(self):
     """Test path whitelisting - empty"""
     testconf = deepcopy(self._conf)
     testconf['whitelist_lvl1'] = ''
     hit = Hit(testconf, u'http://abc.nl/path/to/page')
     self.assertEqual(hit.is_whitelisted(), True)