def test_get_tokens(self): token1 = '<meta name="keywords" content="independent,people"/>' token2 = '<meta name="keywords" content="independent , people"/>' ho = HTMLObject() keys1 = ho.get_keywords_from_meta_tag(token1) self.assertEqual(["independent", "people"], keys1) keys2 = ho.get_keywords_from_meta_tag(token2) self.assertEqual(["independent", "people"], keys2)
def get_webpage_as_object(self, url): try: html, status = self.get_html(url) if html and status == 200: tokeniser = Tokens() tokens = tokeniser.tokenise(html) objectifier = HTMLObject() objectifier.tokens_to_html_object(tokens, url) return objectifier else: return Website() except: return Website()
def test_tokens_to_html_object_with_body_html(self): t = Tokens() tokens = t.tokenise(html) ho = HTMLObject() ho.tokens_to_html_object(tokens) print("done")
def test_tokens_to_html_object_with_title(self): tokens = ["asdf", "<title>", "My title", "</title>"] ho = HTMLObject() ho.tokens_to_html_object(tokens) print("done")