def get_news_posts_by_page(): str = '' try: url_page = request.args.get('page_url') news_posts = NewsPost.query(NewsPost.source_url == url_page).fetch() str += 'num news posts: %d\n' % len(news_posts) dict_titles = {} for np in news_posts: #str += 'title: %s\n' % np.title #str += 'url: %s\n' % np.url #str +='-------\n' dict_titles.setdefault(np.title, []) dict_titles[np.title].append(np.url) for title in dict_titles: str += 'title: %s\n' % title str += 'source(s):\n' for source in dict_titles[title]: str += '%s\n' % source str += '---------\n' except Exception as inst: str += 'Exception: %s' % inst.message return Response(str, mimetype='text/plain')
def take_all_news_posts(): """ :return returns every news post there is: """ return NewsPost.query().fetch()
def getNewsFromSource(): source = request.args.get('source') s = '' newsPosts = NewsPost.query(NewsPost.host_page == source).fetch() for np in newsPosts: s+= '%s\n' % np.title return Response(s, mimetype='text/plain')
def getMYNews(): category = request.args.get('category') news = NewsPost.query().fetch() clusters = Cluster.query(Cluster.category == category).fetch() newNews = [] for n in news: newObject = NewsPostClient(url = n.url,host_page = n.host_page,title = n.title, description = n.description) newNews.append(newObject) #result = str(byteify(newNews[0].serialize())) result = '' return Response(result, mimetype='application/javascript')
def get_posts_by_page(): str = '' url_page = request.args.get('page_url') str += 'url_page: %s\n' % url_page news_posts = NewsPost.query(NewsPost.source_url == url_page).fetch() str += 'num news posts: %d\n' % len(news_posts) light_version = [] for np in news_posts: newNews = NewsPostClient(url = np.url, host_page = np.host_page, title = np.title, numWords = np.numWords, source_id = np.source_id, source_url = np.source_url,img_url = np.img_url, pub_date = np.pub_date, description = np.description) light_version.append(newNews) obj = {'listNewsPosts' :[n.serialize() for n in light_version]} result = json.dumps(obj, ensure_ascii=True) return Response(result, mimetype='text/plain')
def getNewsPosts(source_object, web_page_url, dict_IDF): """ The main function which crawls a particular link and returns news posts as object that have been extracted from that link. Needs revising and (possibly) modifying the process of text extraction. :param sourceObject: the source object that wraps multiple web_page_urls (we need it for creating the news post object) :param web_page_url: the web page url where we extract the information from :param dictIDF: the idf dictionary that we need to calculate tf_idf for a document :return: list of news posts and a feedback (for logging) """ #feedback variable for logging feedback = '' try: #opening the url and reading the content c = urlopen(web_page_url) content = c.read() soup = BeautifulSoup(content) logging.debug('getNewsPosts: failed on reading web_page_url') logging.debug('instantiated beautiful soup') #the list of object that we are going to return newsPosts = [] for item in soup.findAll('item'): #in each item we have a link to the news that we would like to process try: #title of the news title = item.find('title').string #link to the news link_url = item.find('link').string feedback += 'title: %s\n' % title feedback += 'link_url: %s\n' % link_url pub_date = item.find('pubdate') if pub_date is not None: pub_date = pub_date.string datetime_obj = parse(pub_date, ignoretz=True) feedback += 'pub_date: %s\n' % (datetime_obj.strftime('%B %d %Y %H:%M')) date_milli = (datetime_obj - epoch).total_seconds() * 1000.0 pub_date = date_milli feedback += 'milli: %f\n' % date_milli else: pub_date = 0 feedback += 'pub_date: None\n' same_news_posts = NewsPost.query(NewsPost.url == link_url).fetch() #we must not process the same news twice if same_news_posts is not None and len(same_news_posts) > 0: feedback += 'There is/are already news post/s with this link. Continuing..\n' feedback += '------------------------------\n' continue img_url = None #we try to fetch the photo url directly from the rss feed, if not possible we will try later again if (item.description is not None) and (item.description.string is not None): img_obj = BeautifulSoup(item.description.string).find('img') if img_obj is not None: img_url = img_obj['src'] elif item.description is not None: img_obj = item.description.find('img') if img_obj is not None: img_url = img_obj['src'] #here we get the content of the news link_content = urlopen(link_url).read() innerSoup = BeautifulSoup(link_content) title_words = Utility.getWords(title) title_words = filterTitles(title_words, web_page_url) total_words = title_words # add title twice, because we consider those words in the title twice as important as the other words total_words.extend(total_words) #which paragraphs to take into consideration text = '' for script in innerSoup(['script', 'style']): script.extract() if web_page_url in Utility.fetch_text_specifications: specifications = Utility.fetch_text_specifications[web_page_url] if isinstance(specifications, list): #we take the paragraphs start = Utility.fetch_text_specifications[web_page_url][0] end = len(innerSoup.findAll('p')) if len(Utility.fetch_text_specifications[web_page_url]) > 1: end = Utility.fetch_text_specifications[web_page_url][1] for p in innerSoup.findAll('p')[start:end]: text += '%s ' % p.text else: tag_type = specifications['tag_type'] attr_type = specifications['attribute_type'] attr_value = specifications['attribute_value'] sections = innerSoup.findAll(tag_type, {attr_type: attr_value}) if 'nested_tag_type' in specifications: #we need to go one level deeper nested_tag_type = specifications['nested_tag_type'] nested_attr_type = specifications['nested_attribute_type'] nested_attr_value = specifications['nested_attribute_value'] limit = specifications.get('limit', 1000) new_sections = [] for section in sections: new_sections.extend(section.findAll(nested_tag_type,{ nested_attr_type: nested_attr_value}, limit=limit)) sections = new_sections for section in sections: text += '%s ' % section.text description = text[:min(100, len(text))] total_words.extend(Utility.getWords(text)) num_words = len(total_words) if num_words < 7: continue dict_news = {} for word in total_words: dict_news[word] = 1 + dict_news.get(word, 0) #we are trying to get the image from the news if img_url is None: imgs = innerSoup.findAll('img') img_url = '' if imgs is not None and len(imgs) > 0: img_url = imgs[0]['src'] #deal with the pictures with relative path to the web if (img_url is not None) and (len(img_url) > 0): if img_url.find(source_object.url) != 0: img_url = source_object.url + '/' + img_url feedback += 'img_url: %s\n' % img_url newsPost = NewsPost(parent=ndb.Key('NewsPost', link_url or "*notitle*"), url = link_url, host_page = web_page_url, title = title, dictWords = dict_news, numWords = num_words, words = total_words , source_id = source_object.id, source_url = source_object.url, img_url = img_url, pub_date = pub_date, description = description) newsPost.calculate_tf_idf(dict_IDF) newsPost.put() newsPosts.append(newsPost) feedback += '------------------------------\n' except Exception as inst: feedback += 'Inner Exception type: %s\n' % str(type(inst)) feedback += 'Inner Exception message: %s\n' % inst.message return newsPosts, feedback except Exception as inst: feedback += 'Exception type: %s\n' % type(inst) feedback += 'Exception message: %s\n' % inst.message #if there is an exception, we return and empty list of news posts return [], feedback