def test_normalize_url_for_2ch(self):
url = u"""uni.2ch.net/test/read.cgi/newsplus/1333782346/-100"""
self.assertEqual(mm.normalize_url(url), u"""uni.2ch.net/test/read.cgi/newsplus/1333782346/""")
url = u"""uni.2ch.net/test/read.cgi/newsplus/1333782346/100"""
self.assertEqual(mm.normalize_url(url), u"""uni.2ch.net/test/read.cgi/newsplus/1333782346/100""")
url = u"""uni.2ch.net/test/read.cgi/newsplus/1333782346/l50"""
self.assertEqual(mm.normalize_url(url), u"""uni.2ch.net/test/read.cgi/newsplus/1333782346/""")
def test_normalize_url(self):
url = u"""https://www.google.co.jp/#hl=ja&sugexp=frgbld&gs_nf=1&cp=5&gs_id=1i&xhr=t&q=foo+bar&pf=p&sclient=psy-ab&oq=foo+b&aq=&aqi=&aql=&gs_l=&pbx=1&bav=on.2,or.r_gc.r_pw.r_cp.r_qf.,cf.osb&fp=f191a14c9679b78&biw=927&bih=1095"""
self.assertEqual(mm.normalize_url(url), u"""www.google.co.jp/#q=foo+bar""")
url = u"""http://www.google.co.jp/search?client=ubuntu&channel=fs&q=foo+bar&ie=utf-8&oe=utf-8&hl=ja#hl=ja&client=ubuntu&hs=akZ&channel=fs&sclient=psy-ab&q=foo+bar&oq=foo+bar&aq=f&aqi=&aql=&gs_l=serp.3...0l0l0l29454l0l0l0l0l0l0l0l0ll0l0.frgbld.&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&fp=ea013b4510645c1f&biw=962&bih=1026"""
self.assertEqual(mm.normalize_url(url), u"""www.google.co.jp/search?q=foo+bar&q=foo+bar""")
url = u"""http://scholar.google.co.jp/scholar?q=foo+bar&hl=ja&btnG=%E6%A4%9C%E7%B4%A2&lr="""
self.assertEqual(mm.normalize_url(url), u"""scholar.google.co.jp/scholar?q=foo+bar""")
url = u"""http://www.youtube.com/watch?v=mFbkPWu6clM&feature=disco&playnext=1&list=AVTGnpyrBl25xvQpqP7QzSFersQ1EbW-gM"""
self.assertEqual(mm.normalize_url(url), u"""www.youtube.com/watch?v=mFbkPWu6clM""")
url = u"""http://www.youtube.com/results?search_query=thomas&oq=thomas&aq=f&aqi=p-p1g9&aql=&gs_l=youtube-psuggest.3..35i39j0l9.79967l80756l0l80955l6l6l0l0l0l0l88l479l6l6l0."""
self.assertEqual(mm.normalize_url(url), u"""www.youtube.com/results?search_query=thomas""")
url = u"""twitter.com/#!/tos_kamiya/123"""
self.assertEqual(mm.normalize_url(url), u"""twitter.com/tos_kamiya/123""")