def test_tidy_url(self):
data = [
("http://menmedia.co.uk/asiannews/news/crime/s/1420665_man-wanted-in-connection-with-robbery-and-assault?rss=yes",
"http://menmedia.co.uk/asiannews/news/crime/s/1420665_man-wanted-in-connection-with-robbery-and-assault"),
("http://www.belfasttelegraph.co.uk/news/health/diabetes-lsquocan-be-reversed-through-low-calorie-dietrsquo-16015584.html?r=RSS",
"http://www.belfasttelegraph.co.uk/news/health/diabetes-lsquocan-be-reversed-through-low-calorie-dietrsquo-16015584.html"),
("http://nocruft.com/wibble-pibble","http://nocruft.com/wibble-pibble"),
]
for url,tidied in data:
self.assertEqual(ScraperUtils.tidy_url(url), tidied)
