def test_generate_args_output(self):
prefix = 'work4'
# Generate a link that won't appear in the results (wrong prefix)
Link.shorten(long_url='http://www.work4labs.com',
prefix='work4labs').save()
# Generate the link that will be used in the output
link = Link.shorten(long_url='http://www.work4labs.com',
prefix=prefix).save()
header_row = 'hash,long_url,nb_of_clicks'
first_row = '%s,%s,0' % (link.hash, link.long_url)
with io.BytesIO() as stdout, io.BytesIO() as stderr:
management.call_command('urlstats',
prefix,
stdout=stdout,
stderr=stderr)
stdout.seek(0)
stderr.seek(0)
self.assertEqual(stderr.read(), '')
self.assertEqual(stdout.read(),
'%s\r\n%s\r\n' % (header_row, first_row))
def test_shorten_multiple(self):
# Generate a couple of links that we should *not* fall on
Link.shorten("http://www.work4labs.com/", prefix="foobarblah")
Link.shorten("http://www.work4labs.com/", prefix="fooba")
link1 = Link.shorten("http://www.work4labs.com/", prefix="foobar")
self.assertIn("foobar/", link1.hash)
def test_create_random(self):
link1 = Link.create_with_random_short_path(self.URL, "foo")
self.assertEqual(link1.long_url, self.URL)
# pylint: disable=W0612
for iteration in xrange(1, 10):
# We loop 10 times in hopes of encountering an invalid short path
Link.create_with_random_short_path(self.URL, "foo")
def test_find_for_p_and_l_u(self):
prefix = "ole"
long_url = "http://www.work4labs.com/"
Link.shorten(long_url, prefix=prefix)
self.assertEqual(
Link.find_for_prefix_and_long_url(prefix, long_url).explain()["cursor"], "BtreeCursor long_url_hashed"
)
def main(request, path):
'''
Search for a long link matching the `path` and redirect
'''
if len(path) and path[-1] == '/':
# Removing trailing slash so "/jobs/" and "/jobs" redirect identically
path = path[:-1]
link = Link.find_by_hash(path)
if link is None:
# Try to find a matching short link by removing valid "catchall" suffixes
path_prefix, redirect_suffix = suffix_catchall.get_hash_from(path)
if redirect_suffix is not None:
# If we found a suffix, we try to find a link again with the prefix
link = Link.find_by_hash(path_prefix)
else:
redirect_suffix = None
# Instrumentation
prefix_tag = 'prefix:' + link.prefix if link else 'Http404'
statsd.increment('workforus.clicks', tags=[prefix_tag])
statsd.set('workforus.unique_links',
link.hash if link else 'Http404',
tags=[prefix_tag])
statsd.set('workforus.unique_ips',
get_client_ip(request),
tags=['browser:' + get_browser(request)])
# 404 if link not found or register a click if the DB is not in readonly mode
if link is None:
raise Http404
elif not settings.SITE_READ_ONLY:
link.click()
# Tweak the redirection link based on the query string, redirection suffix, etc.
# FIXME: Handle multiple parameters with the same name in the `url`
query = request.GET.copy()
if redirect_suffix is not None:
query[REDIRECT_PARAM_NAME] = redirect_suffix
if bool(query) and REF_PARAM_NAME not in query:
# If we specify a non empty query, indicate that the shortener tweaked the url
query[REF_PARAM_NAME] = REF_PARAM_DEFAULT_VALUE
target_url = url_append_parameters(
link.long_url,
params_to_replace=query,
defaults={REF_PARAM_NAME: REF_PARAM_DEFAULT_VALUE})
# Either redirect the user, or load the target page and display it directly
return (proxy if link.act_as_proxy else redirect)(target_url)
def main(request, path):
"""Search for a long link matching the `path` and redirect"""
path = _extract_valid_path(path)
link = Link.find_by_hash(path)
redirect_suffix = None
if link is None:
# Try to find a matching prefix
parts = path.split('/', 1)
if len(parts) == 2:
path_prefix, redirect_suffix = parts
# If there was a prefix, we try to find a link again
link = Link.find_by_hash(path_prefix)
# Instrumentation
prefix_tag = 'prefix:' + link.prefix if link else 'Http404'
statsd.increment('workforus.clicks', tags=[prefix_tag])
statsd.set('workforus.unique_links', link.hash if link else 'Http404', tags=[prefix_tag])
statsd.set('workforus.unique_ips', get_client_ip(request), tags=['browser:' + get_browser(request)])
# 404 if link not found or register a click if the DB is not in readonly mode
if link is None:
raise Http404
elif mongoengine_is_primary():
link.click()
# Tweak the redirection link based on the query string, redirection suffix, etc.
# FIXME: Handle multiple parameters with the same name in the `url`
query = request.GET.copy()
if redirect_suffix is not None:
query[REDIRECT_PARAM_NAME] = redirect_suffix
if bool(query) and REF_PARAM_NAME not in query:
# If we specify a non empty query, indicate that the shortener tweaked the url
query[REF_PARAM_NAME] = REF_PARAM_DEFAULT_VALUE
target_url = url_append_parameters(
link.long_url,
params_to_replace=query,
defaults={REF_PARAM_NAME: REF_PARAM_DEFAULT_VALUE}
)
# Either redirect the user, or load the target page and display it directly
if link.act_as_proxy:
return proxy(target_url)
return redirect(target_url, permanent=True)
def shorten_twice(self, **kwargs):
kwargs["long_url"] = "http://www.work4labs.com/"
# statsd.histogram should only be created at creation
with patch("django_short_urls.models.statsd") as mock_statsd:
link1 = Link.shorten(**kwargs)
mock_statsd.histogram.assert_called_once()
link2 = Link.shorten(**kwargs)
mock_statsd.histogram.assert_called_once()
self.assertEqual(link1.hash, link2.hash)
def main(request, path):
'''
Search for a long link matching the `path` and redirect
'''
if len(path) and path[-1] == '/':
# Removing trailing slash so "/jobs/" and "/jobs" redirect identically
path = path[:-1]
link = Link.find_by_hash(path)
if link is None:
# Try to find a matching short link by removing valid "catchall" suffixes
path_prefix, redirect_suffix = suffix_catchall.get_hash_from(path)
if redirect_suffix is not None:
# If we found a suffix, we try to find a link again with the prefix
link = Link.find_by_hash(path_prefix)
else:
redirect_suffix = None
# Instrumentation
prefix_tag = 'prefix:' + link.prefix if link else 'Http404'
statsd.increment('workforus.clicks', tags=[prefix_tag])
statsd.set('workforus.unique_links', link.hash if link else 'Http404', tags=[prefix_tag])
statsd.set('workforus.unique_ips', get_client_ip(request), tags=['browser:' + get_browser(request)])
# 404 if link not found or register a click if the DB is not in readonly mode
if link is None:
raise Http404
elif mongoengine_is_primary():
link.click()
# Tweak the redirection link based on the query string, redirection suffix, etc.
# FIXME: Handle multiple parameters with the same name in the `url`
query = request.GET.copy()
if redirect_suffix is not None:
query[REDIRECT_PARAM_NAME] = redirect_suffix
if bool(query) and REF_PARAM_NAME not in query:
# If we specify a non empty query, indicate that the shortener tweaked the url
query[REF_PARAM_NAME] = REF_PARAM_DEFAULT_VALUE
target_url = url_append_parameters(
link.long_url,
params_to_replace=query,
defaults={REF_PARAM_NAME: REF_PARAM_DEFAULT_VALUE}
)
# Either redirect the user, or load the target page and display it directly
return (proxy if link.act_as_proxy else redirect)(target_url)
def test_valid_short_path(self):
self.assertEqual(Link.is_valid_random_short_path("ab2cd"), True)
self.assertEqual(Link.is_valid_random_short_path("ab2"), True)
self.assertEqual(Link.is_valid_random_short_path("a234r434g43gb32r"), True)
self.assertEqual(Link.is_valid_random_short_path("4a"), True)
self.assertEqual(Link.is_valid_random_short_path("abcd"), False)
self.assertEqual(Link.is_valid_random_short_path("ge"), False)
self.assertEqual(Link.is_valid_random_short_path("crap"), False)
self.assertEqual(Link.is_valid_random_short_path("crap42"), False)
self.assertEqual(Link.is_valid_random_short_path("abe4abe"), False)
def find_for_prefix(self, prefix):
# First check that there are no links for this prefix
self.assertEqual(len(Link.find_for_prefix(prefix)), 0)
# Create a link with this prefix and another with another prefix
true_link = Link.shorten("http://www.work4labs.com/", prefix=prefix)
# other link
Link.shorten("http://www.work4labs.com/", prefix="other_%s" % prefix)
# We should only find the true link
links = Link.find_for_prefix(prefix)
self.assertEqual(len(links), 1)
self.assertEqual(links.first().hash, true_link.hash)
def handle(self, *url_prefixes, **options):
self.writer = csv.writer(self.stdout)
self.writer.writerow(['hash', 'long_url', 'nb_of_clicks'])
for prefix in url_prefixes:
for link in Link.find_for_prefix(prefix):
self.writer.writerow([link.hash, link.long_url, link.clicks])
def setUp(self):
self.setting_backup = settings.SITE_READ_ONLY
settings.SITE_READ_ONLY = True
self.factory = RequestFactory()
self.path = 'test42'
self.link = Link.shorten('http://www.work4.com/jobs', short_path=self.path)
def test_create_random_hash_conflict(self, mock_sha1, mock_histogram):
"""Check that we try again if we encounter hash conflicts"""
link1 = Link.create_with_random_short_path(self.URL, 'foo')
url2 = self.URL + '?foo'
link2 = Link.create_with_random_short_path(url2, 'foo')
self.assertEqual(link1.long_url, self.URL)
self.assertEqual(link1.hash, 'foo/5')
self.assertEqual(link2.long_url, url2)
self.assertEqual(link2.hash, 'foo/19')
self.assertEqual(mock_sha1.call_count, 2)
mock_histogram.assert_has_calls([
# First link gets generated in 1 try
call('workforus.nb_tries_to_generate', 1, tags=['prefix:foo']),
# Second one needs 2 tries
call('workforus.nb_tries_to_generate', 2, tags=['prefix:foo'])
])
def test_generate_args_output(self, mock_statsd): # pylint: disable=unused-argument
prefix = 'work4'
# Generate a link that won't appear in the results (wrong prefix)
Link.shorten(long_url='http://www.work4labs.com', prefix='work4labs').save()
# Generate the link that will be used in the output
link = Link.shorten(long_url='http://www.work4labs.com', prefix=prefix).save()
header_row = 'hash,long_url,nb_of_clicks'
first_row = '%s,%s,0' % (link.hash, link.long_url)
with io.BytesIO() as stdout, io.BytesIO() as stderr:
management.call_command('urlstats', prefix, stdout=stdout, stderr=stderr)
stdout.seek(0)
stderr.seek(0)
self.assertEqual(stderr.read(), '')
self.assertEqual(stdout.read(), '%s\r\n%s\r\n' % (header_row, first_row))
def setUp(self):
self.setting_backup = settings.SITE_READ_ONLY
settings.SITE_READ_ONLY = True
self.factory = RequestFactory()
self.path = 'test42'
self.link = Link.shorten('http://www.work4.com/jobs',
short_path=self.path)
def new(request):
'''
Create a new short url based on the POST parameters
'''
if 'login' in request.REQUEST and 'api_key' in request.REQUEST:
login = request.REQUEST['login']
api_key = request.REQUEST['api_key']
user = User.objects(login=login, api_key=api_key).first()
else:
user = None
if user is None:
return response(status=HTTP_UNAUTHORIZED, message="Invalid credentials.")
params = {}
if 'long_url' in request.REQUEST:
params['long_url'] = request.REQUEST['long_url']
(is_valid, error_message) = validate_url(params['long_url'])
else:
(is_valid, error_message) = (False, "Missing parameter: 'long_url'")
if not is_valid:
return response(status=HTTP_BAD_REQUEST, message=error_message)
allow_slashes_in_prefix = 'allow_slashes_in_prefix' in request.REQUEST
for key in ['short_path', 'prefix']:
if key in request.REQUEST:
params[key] = request.REQUEST[key]
if '/' in params[key] and not (key == 'prefix' and allow_slashes_in_prefix):
return response(
status=HTTP_BAD_REQUEST,
message="%s may not contain a '/' character." % key)
try:
link = Link.shorten(**params)
getLogger('app').info('Successfully shortened %s into %s for user %s', link.long_url, link.hash, login)
except ShortPathConflict, err:
del params['short_path'], params['long_url']
if 'prefix' in params:
del params['prefix']
params['hash'] = err.link.hash
return response(status=HTTP_CONFLICT, message=str(err), **params)
def new(request):
'''
Create a new short url based on the POST parameters
'''
if 'login' in request.REQUEST and 'api_key' in request.REQUEST:
login = request.REQUEST['login']
api_key = request.REQUEST['api_key']
user = User.objects(login=login, api_key=api_key).first()
else:
user = None
if user is None:
return response(status=HTTP_UNAUTHORIZED,
message="Invalid credentials.")
params = {}
if 'long_url' in request.REQUEST:
params['long_url'] = request.REQUEST['long_url']
(is_valid, error_message) = validate_url(params['long_url'])
else:
(is_valid, error_message) = (False, "Missing parameter: 'long_url'")
if not is_valid:
return response(status=HTTP_BAD_REQUEST, message=error_message)
for key in ['short_path', 'prefix']:
if key in request.REQUEST:
params[key] = request.REQUEST[key]
if '/' in params[key]:
return response(status=HTTP_BAD_REQUEST,
message="%s may not contain a '/' character." %
key)
try:
link = Link.shorten(**params)
getLogger('app').info('Successfully shortened %s into %s for user %s',
link.long_url, link.hash, login)
except ShortPathConflict, err:
del params['short_path'], params['long_url']
if 'prefix' in params:
del params['prefix']
params['hash'] = err.link.hash
return response(status=HTTP_CONFLICT, message=str(err), **params)
def test_find_for_p_and_l_u(self):
prefix = 'ole'
long_url = "http://www.work4labs.com/"
_shorten(long_url, prefix=prefix)
explanation = Link.find_for_prefix_and_long_url(prefix, long_url).explain(False)
if 'cursor' in explanation: # pragma: no cover
# Mongo 2.x
self.assertEqual(
explanation['cursor'],
u'BtreeCursor long_url_hashed')
else: # pragma: no cover
self.assertEqual(explanation['queryPlanner']['winningPlan']['inputStage']['indexName'], 'long_url_hashed')
def new(request):
"""Create a new short url based on the POST parameters"""
long_url = request.GET.get('long_url')
if long_url is None:
is_valid, error_message = False, "Missing GET parameter: 'long_url'"
else:
is_valid, error_message = validate_url(long_url)
if not is_valid:
return pyw4c_response(status=HTTP_BAD_REQUEST, message=error_message)
params = {}
for key in ['short_path', 'prefix']:
params[key] = request.GET.get(key)
if key == 'prefix' and 'allow_slashes_in_prefix' in request.GET:
continue
if params[key] is not None and '/' in params[key]:
return pyw4c_response(
status=HTTP_BAD_REQUEST,
message="%s may not contain a '/' character." % key)
statsd.increment(
'workforus.new',
tags=['prefix:' + unicode(params['prefix']), 'is_secure:' + unicode(request.is_secure())])
try:
link = Link.shorten(long_url, **params)
getLogger('app').info(
'Successfully shortened %s into %s for user %s',
link.long_url, link.hash, request.user.login)
except ShortPathConflict, err:
del params['short_path'], long_url
del params['prefix']
params['hash'] = err.hash
return pyw4c_response(status=HTTP_CONFLICT, message=str(err), **params)
def setUp(self):
self.factory = RequestFactory()
self.path = 'test42'
self.link = Link.shorten('http://www.work4.com/jobs', short_path=self.path)
def test_create_random(self, mock_histogram):
self.assertEqual(
Link.create_with_random_short_path(self.URL, 'foo').long_url,
self.URL)
mock_histogram.assert_called_once_with('workforus.nb_tries_to_generate', 1, tags=['prefix:foo'])
def _shorten(*args, **kwargs):
with patch('django_short_urls.models.statsd') as mock_statsd:
return Link.shorten(*args, **kwargs), mock_statsd
def test_create_random_is_invalid(self, mock_is_valid, mock_histogram):
"""Check that we try a second time if we randomly generate an invalid hash, and that we count tries properly"""
self.assertEqual(Link.create_with_random_short_path(self.URL, 'foo').long_url, self.URL)
self.assertEqual(mock_is_valid.call_count, 2)
mock_histogram.assert_called_once_with('workforus.nb_tries_to_generate', 2, tags=['prefix:foo'])
def test_shorten_with_short_path(self):
link = Link.shorten("http://www.work4labs.com/", short_path="FooBar")
self.assertEqual(link.hash, "foobar")
def test_create_random_hash_overflow(self, mock_sha1, mock_histogram):
"""Check that we generate a new hash if we exceed the length of the current hash"""
self.assertEqual(Link.create_with_random_short_path(self.URL, 'foo').long_url, self.URL)
self.assertEqual(mock_sha1.call_count, 2)
mock_histogram.assert_has_calls([call('workforus.nb_tries_to_generate', 1, tags=['prefix:foo'])])
def test_shorten(self):
link = Link.shorten("http://www.work4labs.com/")
self.assertIn("work4labs", str(link))
self.assertNotIn("/", link.hash)
def test_valid_short_path(self):
self.assertTrue(Link._is_valid_random_short_path("ab2cd"))
self.assertTrue(Link._is_valid_random_short_path("ab2"))
self.assertTrue(Link._is_valid_random_short_path("a234r434g43gb32r"))
self.assertTrue(Link._is_valid_random_short_path("4a"))
self.assertTrue(Link._is_valid_random_short_path("ge"))
self.assertTrue(Link._is_valid_random_short_path("42"))
self.assertTrue(Link._is_valid_random_short_path("j"))
self.assertFalse(Link._is_valid_random_short_path("42abcd"), 'More than 2 consecutive letters')
self.assertFalse(Link._is_valid_random_short_path("crap"), 'More than 2 consecutive letters')
self.assertFalse(Link._is_valid_random_short_path("crap42"), 'More than 2 consecutive letters')
self.assertFalse(Link._is_valid_random_short_path("abe4abe"), 'More than 2 consecutive letters')
self.assertFalse(Link._is_valid_random_short_path("Jo42"), 'Uppercase used')
def test_prefix(self):
self.assertEqual(Link.shorten(self.URL).prefix, "")
prefix = "foo"
self.assertEqual(Link.shorten(self.URL, prefix=prefix).prefix, prefix)
