def run(self, page_dict): article_xpath = comment_xpaths = post_xpaths = None content = page_dict.get("content") sd = SDAlgorithm() sd.content = content type, article, comments, multiple = sd.analyze_page() if type in ['article', 'comment']: article_xpath = element2path(sd.tree, article.root_node) # create a few variants of the xpath article_xpaths = standardizexpath(sd.tree, article_xpath) #print '\n'.join(str(path) for path in article_xpaths) if type in ['comment']: comment_root_paths = [] for comment in comments: comment_root_paths.append( element2path(sd.tree, comment.root_node)) # look for the regularity in the comments' paths comment_root_paths, common_beg, diff_middle, common_end = stringops.find_difference_inside( comment_root_paths) if diff_middle.isdigit(): #print 'regularity found' commenttemplatepath = common_beg + xpathops.STANDARD_REPLACEMENT_STRING + common_end # create a few variants of the xpath comment_xpaths = findcommonstandardxpath( sd.tree, commenttemplatepath, comment_root_paths) #print '\n'.join(str(path) for path in comment_xpaths) if type in ['multiple']: posts_root_paths = [] for post in multiple: posts_root_paths.append(element2path(sd.tree, post.root_node)) # look for the regularity in the comments' paths posts_root_paths, common_beg, diff_middle, common_end = stringops.find_difference_inside( posts_root_paths) if diff_middle.isdigit(): #print 'regularity found' posttemplatepath = common_beg + xpathops.STANDARD_REPLACEMENT_STRING + common_end # create a few variants of the xpath post_xpaths = findcommonstandardxpath(sd.tree, posttemplatepath, posts_root_paths) #print '\n'.join(str(path) for path in post_xpaths) return article_xpath, comment_xpaths, post_xpaths
def find(self,params): tree = self.page_dict.get("tree") candidates = [] candidates.extend(self.__searchInputNodes()) candidates.extend(self.__searchAnchorNodes()) candidates.extend(self.__searchButtonNodes()) candidates.extend(self.__searchImageNodes()) if candidates: sorted_candidates = sorted(candidates,key=lambda x: -x[1]) # for elem in sorted_candidates: # print elem[0],'->',elem[1] # print html.tostring(elem[0]) # print '#######################################################\n' element = sorted_candidates[0] elements_list = [element] for i in range(1,len(candidates)): if element[1] == candidates[i][1]: elements_list.append(candidates[i]) else: break self.nodepath = [] for elem in elements_list: self.nodepath.append(xpathops.element2path(tree, elem[0])) self.features = [element[1]] self.certainty = 1 if element[1] >= 1 else 0 return self.nodepath else: self.features = [0] self.certainty = 0 self.nodepath = None return self.nodepath
def run(self, page_dict): article_xpath = comment_xpaths = post_xpaths = None content = page_dict.get("content") sd = SDAlgorithm() sd.content = content type,article,comments,multiple = sd.analyze_page() if type in ['article','comment']: article_xpath = element2path(sd.tree,article.root_node) # create a few variants of the xpath article_xpaths = standardizexpath(sd.tree,article_xpath) #print '\n'.join(str(path) for path in article_xpaths) if type in ['comment']: comment_root_paths = [] for comment in comments: comment_root_paths.append(element2path(sd.tree,comment.root_node)) # look for the regularity in the comments' paths comment_root_paths, common_beg, diff_middle, common_end = stringops.find_difference_inside(comment_root_paths) if diff_middle.isdigit(): #print 'regularity found' commenttemplatepath = common_beg+xpathops.STANDARD_REPLACEMENT_STRING+common_end # create a few variants of the xpath comment_xpaths = findcommonstandardxpath(sd.tree,commenttemplatepath,comment_root_paths) #print '\n'.join(str(path) for path in comment_xpaths) if type in ['multiple']: posts_root_paths = [] for post in multiple: posts_root_paths.append(element2path(sd.tree,post.root_node)) # look for the regularity in the comments' paths posts_root_paths, common_beg, diff_middle, common_end = stringops.find_difference_inside(posts_root_paths) if diff_middle.isdigit(): #print 'regularity found' posttemplatepath = common_beg+xpathops.STANDARD_REPLACEMENT_STRING+common_end # create a few variants of the xpath post_xpaths = findcommonstandardxpath(sd.tree,posttemplatepath,posts_root_paths) #print '\n'.join(str(path) for path in post_xpaths) return article_xpath, comment_xpaths, post_xpaths
def classify_page(self): """ Characterize the page according to i) has main article (has_article()), ii) has main article with comments (is_full_article()), iii) has multiple opinions like a forum (is_discussion()). """ validated = False [biggest_regions, grouped_comments] = self.group_regions() [article_exists, article] = self.has_article(biggest_regions) if article_exists: max_group = self.get_candidate_article(article, grouped_comments) if grouped_comments.has_key(max_group): if grouped_comments != {}: validated = self.candidate_group_level_validated( max_group, article, grouped_comments) context_validated = self.candidate_context_validated( article, grouped_comments, max_group) if self.big_areas_in_same_level(article, grouped_comments, max_group) and not validated: print Tcolors.INFO + " Multiple similar regions detected!" print "Class: " print Tcolors.RES + " " + grouped_comments[max_group][ 0].class_name print "Texts: " for reg in grouped_comments[max_group]: print element2path(reg.tree, reg.root_node), reg.full_text return None, None, grouped_comments[max_group] elif not context_validated: print self.print_article(article) print print Tcolors.INFO + " No comments found." return article, None, None elif context_validated: print print Tcolors.INFO + " Article with comments detected!" self.print_article(article) print print "Comment class:" print Tcolors.RES + " " + max_group print "Comments:" for com in grouped_comments[max_group]: print element2path(com.tree, com.root_node), com.full_text return article, grouped_comments[max_group], None else: self.print_article(article) return article, None, None else: print Tcolors.INFO + " Multiple similar regions detected!" print Tcolors.RES print "Texts: " for reg in biggest_regions: print element2path(reg.tree, reg.root_node), reg.full_text return None, None, biggest_regions
def process_multiple_details(self, multiple): #content_list = [] density_list = [] distance_list = [] multiple_article_root_paths = [] for article in multiple: #content_list.append(article.contents) multiple_article_root_paths.append(element2path(self.sd.tree, article.root_node)) density_list.append(article.density) distance_list.append(article.distance_from_root) return multiple_article_root_paths, density_list, distance_list
def process_comment_details(self, comments): #content_list = [] comment_root_paths = [] density_list = [] distance_list = [] for comment in comments: #content_list.append(comment.contents) comment_root_paths.append(element2path(self.sd.tree, comment.root_node)) density_list.append(comment.density) distance_list.append(comment.disance_from_root) return comment_root_paths, density_list, distance_list
def process_multiple_details(self, multiple): #content_list = [] density_list = [] distance_list = [] multiple_article_root_paths = [] for article in multiple: #content_list.append(article.contents) multiple_article_root_paths.append(element2path(self.sd.tree, article.root_node)) density_list.append(article.density) distance_list.append(article.distance_from_root) return multiple_article_root_paths, density_list, distance_list
def process_comment_details(self, comments): #content_list = [] comment_root_paths = [] density_list = [] distance_list = [] for comment in comments: #content_list.append(comment.contents) comment_root_paths.append(element2path(self.sd.tree, comment.root_node)) density_list.append(comment.density) distance_list.append(comment.disance_from_root) return comment_root_paths, density_list, distance_list
def verify_comments(self, comments, tree): comment_root_paths = [] for com in comments: comment_root_paths.append(element2path(tree, com.root_node)) comment_root_paths, common_beg, diff_middle, common_end = stringops.find_difference_inside(comment_root_paths) if diff_middle.isdigit(): print 'Comments: regularity found' return True else: return False
def verify_comments(self, comments, tree): comment_root_paths = [] for com in comments: comment_root_paths.append(element2path(tree, com.root_node)) comment_root_paths, common_beg, diff_middle, common_end = stringops.find_difference_inside( comment_root_paths) if diff_middle.isdigit(): print 'Comments: regularity found' return True else: return False
def verify_multiple_articles(self, mulart, url, tree, content): mul_art_root_paths = [] multiple_article_text = [] for mul in mulart: multiple_article_text.append(mul.full_text.encode('utf-8', 'ignore')) mul_art_root_paths.append(element2path(tree, mul.root_node)) mul_art_root_paths, multiple_article_text = self.leave_roots_only(mul_art_root_paths, multiple_article_text) if(len(mul_art_root_paths)>1): struct_verified = self.verify_multiple_articles_pagetags_structure(mul_art_root_paths) href_verified = self.verify_multiple_article_hrefs(url, content) sim_text_verified = self.verify_similar_text(multiple_article_text, url) if struct_verified and href_verified and sim_text_verified: return True else: return False else: return False # but actually it means : that's an article
def verify_multiple_articles(self, mulart, url, tree, content): mul_art_root_paths = [] multiple_article_text = [] for mul in mulart: multiple_article_text.append( mul.full_text.encode('utf-8', 'ignore')) mul_art_root_paths.append(element2path(tree, mul.root_node)) mul_art_root_paths, multiple_article_text = self.leave_roots_only( mul_art_root_paths, multiple_article_text) if (len(mul_art_root_paths) > 1): struct_verified = self.verify_multiple_articles_pagetags_structure( mul_art_root_paths) href_verified = self.verify_multiple_article_hrefs(url, content) sim_text_verified = self.verify_similar_text( multiple_article_text, url) if struct_verified and href_verified and sim_text_verified: return True else: return False else: return False # but actually it means : that's an article