def test_generate_args_output(self): prefix = 'work4' # Generate a link that won't appear in the results (wrong prefix) Link.shorten(long_url='http://www.work4labs.com', prefix='work4labs').save() # Generate the link that will be used in the output link = Link.shorten(long_url='http://www.work4labs.com', prefix=prefix).save() header_row = 'hash,long_url,nb_of_clicks' first_row = '%s,%s,0' % (link.hash, link.long_url) with io.BytesIO() as stdout, io.BytesIO() as stderr: management.call_command('urlstats', prefix, stdout=stdout, stderr=stderr) stdout.seek(0) stderr.seek(0) self.assertEqual(stderr.read(), '') self.assertEqual(stdout.read(), '%s\r\n%s\r\n' % (header_row, first_row))
def test_shorten_multiple(self): # Generate a couple of links that we should *not* fall on Link.shorten("http://www.work4labs.com/", prefix="foobarblah") Link.shorten("http://www.work4labs.com/", prefix="fooba") link1 = Link.shorten("http://www.work4labs.com/", prefix="foobar") self.assertIn("foobar/", link1.hash)
def test_create_random(self): link1 = Link.create_with_random_short_path(self.URL, "foo") self.assertEqual(link1.long_url, self.URL) # pylint: disable=W0612 for iteration in xrange(1, 10): # We loop 10 times in hopes of encountering an invalid short path Link.create_with_random_short_path(self.URL, "foo")
def test_find_for_p_and_l_u(self): prefix = "ole" long_url = "http://www.work4labs.com/" Link.shorten(long_url, prefix=prefix) self.assertEqual( Link.find_for_prefix_and_long_url(prefix, long_url).explain()["cursor"], "BtreeCursor long_url_hashed" )
def main(request, path): ''' Search for a long link matching the `path` and redirect ''' if len(path) and path[-1] == '/': # Removing trailing slash so "/jobs/" and "/jobs" redirect identically path = path[:-1] link = Link.find_by_hash(path) if link is None: # Try to find a matching short link by removing valid "catchall" suffixes path_prefix, redirect_suffix = suffix_catchall.get_hash_from(path) if redirect_suffix is not None: # If we found a suffix, we try to find a link again with the prefix link = Link.find_by_hash(path_prefix) else: redirect_suffix = None # Instrumentation prefix_tag = 'prefix:' + link.prefix if link else 'Http404' statsd.increment('workforus.clicks', tags=[prefix_tag]) statsd.set('workforus.unique_links', link.hash if link else 'Http404', tags=[prefix_tag]) statsd.set('workforus.unique_ips', get_client_ip(request), tags=['browser:' + get_browser(request)]) # 404 if link not found or register a click if the DB is not in readonly mode if link is None: raise Http404 elif not settings.SITE_READ_ONLY: link.click() # Tweak the redirection link based on the query string, redirection suffix, etc. # FIXME: Handle multiple parameters with the same name in the `url` query = request.GET.copy() if redirect_suffix is not None: query[REDIRECT_PARAM_NAME] = redirect_suffix if bool(query) and REF_PARAM_NAME not in query: # If we specify a non empty query, indicate that the shortener tweaked the url query[REF_PARAM_NAME] = REF_PARAM_DEFAULT_VALUE target_url = url_append_parameters( link.long_url, params_to_replace=query, defaults={REF_PARAM_NAME: REF_PARAM_DEFAULT_VALUE}) # Either redirect the user, or load the target page and display it directly return (proxy if link.act_as_proxy else redirect)(target_url)
def main(request, path): """Search for a long link matching the `path` and redirect""" path = _extract_valid_path(path) link = Link.find_by_hash(path) redirect_suffix = None if link is None: # Try to find a matching prefix parts = path.split('/', 1) if len(parts) == 2: path_prefix, redirect_suffix = parts # If there was a prefix, we try to find a link again link = Link.find_by_hash(path_prefix) # Instrumentation prefix_tag = 'prefix:' + link.prefix if link else 'Http404' statsd.increment('workforus.clicks', tags=[prefix_tag]) statsd.set('workforus.unique_links', link.hash if link else 'Http404', tags=[prefix_tag]) statsd.set('workforus.unique_ips', get_client_ip(request), tags=['browser:' + get_browser(request)]) # 404 if link not found or register a click if the DB is not in readonly mode if link is None: raise Http404 elif mongoengine_is_primary(): link.click() # Tweak the redirection link based on the query string, redirection suffix, etc. # FIXME: Handle multiple parameters with the same name in the `url` query = request.GET.copy() if redirect_suffix is not None: query[REDIRECT_PARAM_NAME] = redirect_suffix if bool(query) and REF_PARAM_NAME not in query: # If we specify a non empty query, indicate that the shortener tweaked the url query[REF_PARAM_NAME] = REF_PARAM_DEFAULT_VALUE target_url = url_append_parameters( link.long_url, params_to_replace=query, defaults={REF_PARAM_NAME: REF_PARAM_DEFAULT_VALUE} ) # Either redirect the user, or load the target page and display it directly if link.act_as_proxy: return proxy(target_url) return redirect(target_url, permanent=True)
def shorten_twice(self, **kwargs): kwargs["long_url"] = "http://www.work4labs.com/" # statsd.histogram should only be created at creation with patch("django_short_urls.models.statsd") as mock_statsd: link1 = Link.shorten(**kwargs) mock_statsd.histogram.assert_called_once() link2 = Link.shorten(**kwargs) mock_statsd.histogram.assert_called_once() self.assertEqual(link1.hash, link2.hash)
def main(request, path): ''' Search for a long link matching the `path` and redirect ''' if len(path) and path[-1] == '/': # Removing trailing slash so "/jobs/" and "/jobs" redirect identically path = path[:-1] link = Link.find_by_hash(path) if link is None: # Try to find a matching short link by removing valid "catchall" suffixes path_prefix, redirect_suffix = suffix_catchall.get_hash_from(path) if redirect_suffix is not None: # If we found a suffix, we try to find a link again with the prefix link = Link.find_by_hash(path_prefix) else: redirect_suffix = None # Instrumentation prefix_tag = 'prefix:' + link.prefix if link else 'Http404' statsd.increment('workforus.clicks', tags=[prefix_tag]) statsd.set('workforus.unique_links', link.hash if link else 'Http404', tags=[prefix_tag]) statsd.set('workforus.unique_ips', get_client_ip(request), tags=['browser:' + get_browser(request)]) # 404 if link not found or register a click if the DB is not in readonly mode if link is None: raise Http404 elif mongoengine_is_primary(): link.click() # Tweak the redirection link based on the query string, redirection suffix, etc. # FIXME: Handle multiple parameters with the same name in the `url` query = request.GET.copy() if redirect_suffix is not None: query[REDIRECT_PARAM_NAME] = redirect_suffix if bool(query) and REF_PARAM_NAME not in query: # If we specify a non empty query, indicate that the shortener tweaked the url query[REF_PARAM_NAME] = REF_PARAM_DEFAULT_VALUE target_url = url_append_parameters( link.long_url, params_to_replace=query, defaults={REF_PARAM_NAME: REF_PARAM_DEFAULT_VALUE} ) # Either redirect the user, or load the target page and display it directly return (proxy if link.act_as_proxy else redirect)(target_url)
def test_valid_short_path(self): self.assertEqual(Link.is_valid_random_short_path("ab2cd"), True) self.assertEqual(Link.is_valid_random_short_path("ab2"), True) self.assertEqual(Link.is_valid_random_short_path("a234r434g43gb32r"), True) self.assertEqual(Link.is_valid_random_short_path("4a"), True) self.assertEqual(Link.is_valid_random_short_path("abcd"), False) self.assertEqual(Link.is_valid_random_short_path("ge"), False) self.assertEqual(Link.is_valid_random_short_path("crap"), False) self.assertEqual(Link.is_valid_random_short_path("crap42"), False) self.assertEqual(Link.is_valid_random_short_path("abe4abe"), False)
def find_for_prefix(self, prefix): # First check that there are no links for this prefix self.assertEqual(len(Link.find_for_prefix(prefix)), 0) # Create a link with this prefix and another with another prefix true_link = Link.shorten("http://www.work4labs.com/", prefix=prefix) # other link Link.shorten("http://www.work4labs.com/", prefix="other_%s" % prefix) # We should only find the true link links = Link.find_for_prefix(prefix) self.assertEqual(len(links), 1) self.assertEqual(links.first().hash, true_link.hash)
def handle(self, *url_prefixes, **options): self.writer = csv.writer(self.stdout) self.writer.writerow(['hash', 'long_url', 'nb_of_clicks']) for prefix in url_prefixes: for link in Link.find_for_prefix(prefix): self.writer.writerow([link.hash, link.long_url, link.clicks])
def setUp(self): self.setting_backup = settings.SITE_READ_ONLY settings.SITE_READ_ONLY = True self.factory = RequestFactory() self.path = 'test42' self.link = Link.shorten('http://www.work4.com/jobs', short_path=self.path)
def test_create_random_hash_conflict(self, mock_sha1, mock_histogram): """Check that we try again if we encounter hash conflicts""" link1 = Link.create_with_random_short_path(self.URL, 'foo') url2 = self.URL + '?foo' link2 = Link.create_with_random_short_path(url2, 'foo') self.assertEqual(link1.long_url, self.URL) self.assertEqual(link1.hash, 'foo/5') self.assertEqual(link2.long_url, url2) self.assertEqual(link2.hash, 'foo/19') self.assertEqual(mock_sha1.call_count, 2) mock_histogram.assert_has_calls([ # First link gets generated in 1 try call('workforus.nb_tries_to_generate', 1, tags=['prefix:foo']), # Second one needs 2 tries call('workforus.nb_tries_to_generate', 2, tags=['prefix:foo']) ])
def test_generate_args_output(self, mock_statsd): # pylint: disable=unused-argument prefix = 'work4' # Generate a link that won't appear in the results (wrong prefix) Link.shorten(long_url='http://www.work4labs.com', prefix='work4labs').save() # Generate the link that will be used in the output link = Link.shorten(long_url='http://www.work4labs.com', prefix=prefix).save() header_row = 'hash,long_url,nb_of_clicks' first_row = '%s,%s,0' % (link.hash, link.long_url) with io.BytesIO() as stdout, io.BytesIO() as stderr: management.call_command('urlstats', prefix, stdout=stdout, stderr=stderr) stdout.seek(0) stderr.seek(0) self.assertEqual(stderr.read(), '') self.assertEqual(stdout.read(), '%s\r\n%s\r\n' % (header_row, first_row))
def new(request): ''' Create a new short url based on the POST parameters ''' if 'login' in request.REQUEST and 'api_key' in request.REQUEST: login = request.REQUEST['login'] api_key = request.REQUEST['api_key'] user = User.objects(login=login, api_key=api_key).first() else: user = None if user is None: return response(status=HTTP_UNAUTHORIZED, message="Invalid credentials.") params = {} if 'long_url' in request.REQUEST: params['long_url'] = request.REQUEST['long_url'] (is_valid, error_message) = validate_url(params['long_url']) else: (is_valid, error_message) = (False, "Missing parameter: 'long_url'") if not is_valid: return response(status=HTTP_BAD_REQUEST, message=error_message) allow_slashes_in_prefix = 'allow_slashes_in_prefix' in request.REQUEST for key in ['short_path', 'prefix']: if key in request.REQUEST: params[key] = request.REQUEST[key] if '/' in params[key] and not (key == 'prefix' and allow_slashes_in_prefix): return response( status=HTTP_BAD_REQUEST, message="%s may not contain a '/' character." % key) try: link = Link.shorten(**params) getLogger('app').info('Successfully shortened %s into %s for user %s', link.long_url, link.hash, login) except ShortPathConflict, err: del params['short_path'], params['long_url'] if 'prefix' in params: del params['prefix'] params['hash'] = err.link.hash return response(status=HTTP_CONFLICT, message=str(err), **params)
def new(request): ''' Create a new short url based on the POST parameters ''' if 'login' in request.REQUEST and 'api_key' in request.REQUEST: login = request.REQUEST['login'] api_key = request.REQUEST['api_key'] user = User.objects(login=login, api_key=api_key).first() else: user = None if user is None: return response(status=HTTP_UNAUTHORIZED, message="Invalid credentials.") params = {} if 'long_url' in request.REQUEST: params['long_url'] = request.REQUEST['long_url'] (is_valid, error_message) = validate_url(params['long_url']) else: (is_valid, error_message) = (False, "Missing parameter: 'long_url'") if not is_valid: return response(status=HTTP_BAD_REQUEST, message=error_message) for key in ['short_path', 'prefix']: if key in request.REQUEST: params[key] = request.REQUEST[key] if '/' in params[key]: return response(status=HTTP_BAD_REQUEST, message="%s may not contain a '/' character." % key) try: link = Link.shorten(**params) getLogger('app').info('Successfully shortened %s into %s for user %s', link.long_url, link.hash, login) except ShortPathConflict, err: del params['short_path'], params['long_url'] if 'prefix' in params: del params['prefix'] params['hash'] = err.link.hash return response(status=HTTP_CONFLICT, message=str(err), **params)
def test_find_for_p_and_l_u(self): prefix = 'ole' long_url = "http://www.work4labs.com/" _shorten(long_url, prefix=prefix) explanation = Link.find_for_prefix_and_long_url(prefix, long_url).explain(False) if 'cursor' in explanation: # pragma: no cover # Mongo 2.x self.assertEqual( explanation['cursor'], u'BtreeCursor long_url_hashed') else: # pragma: no cover self.assertEqual(explanation['queryPlanner']['winningPlan']['inputStage']['indexName'], 'long_url_hashed')
def new(request): """Create a new short url based on the POST parameters""" long_url = request.GET.get('long_url') if long_url is None: is_valid, error_message = False, "Missing GET parameter: 'long_url'" else: is_valid, error_message = validate_url(long_url) if not is_valid: return pyw4c_response(status=HTTP_BAD_REQUEST, message=error_message) params = {} for key in ['short_path', 'prefix']: params[key] = request.GET.get(key) if key == 'prefix' and 'allow_slashes_in_prefix' in request.GET: continue if params[key] is not None and '/' in params[key]: return pyw4c_response( status=HTTP_BAD_REQUEST, message="%s may not contain a '/' character." % key) statsd.increment( 'workforus.new', tags=['prefix:' + unicode(params['prefix']), 'is_secure:' + unicode(request.is_secure())]) try: link = Link.shorten(long_url, **params) getLogger('app').info( 'Successfully shortened %s into %s for user %s', link.long_url, link.hash, request.user.login) except ShortPathConflict, err: del params['short_path'], long_url del params['prefix'] params['hash'] = err.hash return pyw4c_response(status=HTTP_CONFLICT, message=str(err), **params)
def setUp(self): self.factory = RequestFactory() self.path = 'test42' self.link = Link.shorten('http://www.work4.com/jobs', short_path=self.path)
def test_create_random(self, mock_histogram): self.assertEqual( Link.create_with_random_short_path(self.URL, 'foo').long_url, self.URL) mock_histogram.assert_called_once_with('workforus.nb_tries_to_generate', 1, tags=['prefix:foo'])
def _shorten(*args, **kwargs): with patch('django_short_urls.models.statsd') as mock_statsd: return Link.shorten(*args, **kwargs), mock_statsd
def test_create_random_is_invalid(self, mock_is_valid, mock_histogram): """Check that we try a second time if we randomly generate an invalid hash, and that we count tries properly""" self.assertEqual(Link.create_with_random_short_path(self.URL, 'foo').long_url, self.URL) self.assertEqual(mock_is_valid.call_count, 2) mock_histogram.assert_called_once_with('workforus.nb_tries_to_generate', 2, tags=['prefix:foo'])
def test_shorten_with_short_path(self): link = Link.shorten("http://www.work4labs.com/", short_path="FooBar") self.assertEqual(link.hash, "foobar")
def test_create_random_hash_overflow(self, mock_sha1, mock_histogram): """Check that we generate a new hash if we exceed the length of the current hash""" self.assertEqual(Link.create_with_random_short_path(self.URL, 'foo').long_url, self.URL) self.assertEqual(mock_sha1.call_count, 2) mock_histogram.assert_has_calls([call('workforus.nb_tries_to_generate', 1, tags=['prefix:foo'])])
def test_shorten(self): link = Link.shorten("http://www.work4labs.com/") self.assertIn("work4labs", str(link)) self.assertNotIn("/", link.hash)
def test_valid_short_path(self): self.assertTrue(Link._is_valid_random_short_path("ab2cd")) self.assertTrue(Link._is_valid_random_short_path("ab2")) self.assertTrue(Link._is_valid_random_short_path("a234r434g43gb32r")) self.assertTrue(Link._is_valid_random_short_path("4a")) self.assertTrue(Link._is_valid_random_short_path("ge")) self.assertTrue(Link._is_valid_random_short_path("42")) self.assertTrue(Link._is_valid_random_short_path("j")) self.assertFalse(Link._is_valid_random_short_path("42abcd"), 'More than 2 consecutive letters') self.assertFalse(Link._is_valid_random_short_path("crap"), 'More than 2 consecutive letters') self.assertFalse(Link._is_valid_random_short_path("crap42"), 'More than 2 consecutive letters') self.assertFalse(Link._is_valid_random_short_path("abe4abe"), 'More than 2 consecutive letters') self.assertFalse(Link._is_valid_random_short_path("Jo42"), 'Uppercase used')
def test_prefix(self): self.assertEqual(Link.shorten(self.URL).prefix, "") prefix = "foo" self.assertEqual(Link.shorten(self.URL, prefix=prefix).prefix, prefix)