def post(self): post = self.request.POST if post['kind'] == 'badge': badge = Badge(name=post['name'], description=post['description'], category=post['category'], image=post['image'], value=int(post['value'])) badge.save() elif post['kind'] == 'article': date = datetime.datetime.strptime(post['date'], '%Y-%m-%d').date() article = NewsArticle(title=post['title'], author=post['author'], body=post['body'], date=date) article.save() elif post['kind'] == 'award': badge = Badge.get_by_id(int(post['badge'])) for member in post.getall('members'): member = Member.get_by_id(int(member)) award = Award(member=member, badge=badge, date=datetime.date.today(), proof=post['proof']) award.save() elif post['kind'] == 'talk': talk = Talk(title=post['title'], date=datetime.datetime.strptime( post['date'], '%Y-%m-%d').date(), description=post['description'], member=Member.get_by_id(int(post['member'])), video=post['video']) talk.put() self.get()
def get_gap_fill_questions(): url = request.args.get('url') limit = int(request.args.get('limit') or 50) article_text = get_article_text(url) questions = generate_gap_fill_questions(article_text) num_questions = len(questions) for key in ["_type", "_subtype"]: for q in questions: q.pop(key) try: db.session.add( QuestionGenRequest(url=url, questions=questions, question_type="gap_fill")) db.session.commit() except Exception as e: print(e) try: parsed_url = urlparse(url) article_id = md5(article_text.strip().encode('utf-8')) if not NewsArticle.query.get(article_id.hexdigest()): db.session.add( NewsArticle(id=article_id.hexdigest(), url=url, article_text=article_text.strip(), domain=parsed_url.netloc)) db.session.commit() except Exception as e: print(e) for q in questions: q_id = md5(q.get('question').encode('utf-8')) if not Question.query.get(q_id.hexdigest()): try: db.session.add( Question(id=q_id.hexdigest(), question_text=q.get("question"), source_sentence=q.get("source_sentence"), correct_answer=q.get("answer"), good_question_votes=0, bad_question_votes=0, news_article_id=article_id.hexdigest())) db.session.commit() except Exception as e: print("Unable to add item to database.") print(e) news_article = NewsArticle.query.get(article_id.hexdigest()) questions = question_schema.dump(news_article.questions.all()).data shuffle(questions) return jsonify({ 'num_questions': min(limit, num_questions), 'questions': questions[:limit] })
def newsarticle_create(username: str): article = NewsArticle( link = "test link", wing = "test wing", text = "some sample text" ).save() return article
def post(self): post = self.request.POST if "delete_article" in post: output = post.getall('delete_article') try: counter = 0 for articleKey in output: article = NewsArticle.get(Key(articleKey)) article.delete() counter += 1 message = '%d Article(s) deleted.' % counter except datastore_errors.Error: message = 'Database error, a subset of your selection might have been deleted' else: message = 'No articles selected, congratulations, you wasted some CPU time.' news_list = NewsArticle.all().order('-date'); self.render_template('admin_news', { 'news_list' : news_list, 'delete_successful' : message })
def post(self): articles_deleted = 0 for article_key in self.request.POST.getall('delete_article'): try: article_key = Key(article_key) except BadKeyError: # Wrong syntax for a key, move on to next key. continue article = NewsArticle.get(article_key) if article: article.delete() articles_deleted += 1 # Else, not article has this key. self.render_template('admin_news', {'news_list': NewsArticle.all().order('-date'), 'delete_successful': '%d article(s) deleted.' % articles_deleted})
def post(self): post = self.request.POST if post['kind'] == 'badge': b = Badge(name=post['name'], description=post['description'], category=post['category'], image=post['image'], value=int(post['value'])) b.save() elif post['kind'] == 'article': a = NewsArticle(title=post['title'], author=post['author'], body=post['body'], date=datetime.date.today()) a.save() elif post['kind'] == 'award': badge = Badge.gql('WHERE name = :1', post['badge']).get() for h in post.getall('hackers'): hacker = Hacker.gql('WHERE handle = :1', h).get() a = Award(hacker=hacker, badge=badge, date=datetime.date.today(), proof=post['proof']) a.save() hacker.score_cache = hacker.score + badge.value hacker.save() self.get()
def post(self): post = self.request.POST if post['kind'] == 'badge': badge = Badge( name=post['name'], description=post['description'], category=post['category'], image=post['image'], value=int(post['value']) ) badge.save() elif post['kind'] == 'article': date = datetime.datetime.strptime(post['date'], '%Y-%m-%d').date() article = NewsArticle( title=post['title'], author=post['author'], body=post['body'], date=date ) article.save() elif post['kind'] == 'award': badge = Badge.get_by_id(int(post['badge'])) for member in post.getall('members'): member = Member.get_by_id(int(member)) award = Award( member=member, badge=badge, date=datetime.date.today(), proof=post['proof'] ) award.save() member.score_cache = member.score + badge.value member.save() elif post['kind'] == 'talk': talk = Talk( title=post['title'], date=datetime.datetime.strptime(post['date'], '%Y-%m-%d').date(), description=post['description'], member=Member.get_by_id(int(post['member'])), video=post['video'] ) talk.put() self.get()
def get_rows_to_create(json_data_from_file: Dict) -> List[NewsArticle]: rows_to_add = list() len_of_rows = len(json_data_from_file) print(f"Total rows to import {len_of_rows}") for index, json_data in enumerate(json_data_from_file): print(f"Import {index} of {len_of_rows}") _preprocess_data(json_data) new_row = NewsArticle(**json_data) rows_to_add.append(new_row) return rows_to_add
def post(self): post = self.request.POST if "delete_article" in post: output = post.getall('delete_article') try: counter = 0 for articleKey in output: article = NewsArticle.get(Key(articleKey)) article.delete() counter += 1 message = '%d Article(s) deleted.' % counter except datastore_errors.Error: message = 'Database error, a subset of your selection might have been deleted' else: message = 'No articles selected, congratulations, you wasted some CPU time.' news_list = NewsArticle.all().order('-date') self.render_template('admin_news', { 'news_list': news_list, 'delete_successful': message })
def get(self, key): template_dict = {'key': key, 'show_form' : True} if key == 'new': template_dict['form_data'] = { 'title': '', 'author': Member.get_current_member().handle, 'date': datetime.date.today(), 'body': ''} else: try: template_dict['form_data'] = NewsArticle.get(Key(key)) except BadKeyError: template_dict['message'] = \ 'Could not find article with key %r.' % key template_dict['show_form'] = False self.render_template('edit', template_dict)
def get(self, key): template_dict = {'key': key, 'show_form': True} if key == 'new': template_dict['form_data'] = { 'title': '', 'author': Member.get_current_member().handle, 'date': datetime.date.today(), 'body': '' } else: try: template_dict['form_data'] = NewsArticle.get(Key(key)) except BadKeyError: template_dict['message'] = \ 'Could not find article with key %r.' % key template_dict['show_form'] = False self.render_template('edit', template_dict)
def addNews(): data = request.json news = News() for article in data['articles']: newArticle = NewsArticle(title=article['title'], description=article['description'], lang=article['lang']) news.articles.append(newArticle) db.session.add(news) db.session.commit() return Response(json.dumps( {'message': 'News has been created succesfully'}), status=200, mimetype='application/json')
def post(self): post = self.request.POST if post['kind'] == 'badge': b = Badge(name=post['name'], description=post['description'], category=post['category'], image=post['image'], value=int(post['value'])) b.save() elif post['kind'] == 'article': a = NewsArticle(title=post['title'], author=post['author'], body=post['body'], date=datetime.date.today()) a.save() elif post['kind'] == 'award': badge = Badge.gql('WHERE name = :1', post['badge']).get() for h in post.getall('hackers'): hacker = Hacker.gql('WHERE handle = :1', h).get() a=Award(hacker=hacker, badge=badge, date=datetime.date.today(), proof=post['proof']) a.save() hacker.score_cache = hacker.score + badge.value hacker.save() self.get()
def get(self): self.render_template('admin_news', {'news_list' : NewsArticle.all().order('-date')})
def post(self, key): post = self.request.POST form_data = dict( (k, post.get(k, '')) for k in ('title', 'author', 'date', 'body')) template_dict = {'form_data': form_data, 'key': key, 'show_form': True} if 'delete_article' in post: try: NewsArticle.get(Key(key)).delete() except datastore_errors.Error: template_dict['message'] = \ 'Could not delete article with key %r.' % key else: template_dict['message'] = 'Article deleted.' template_dict['show_form'] = False else: try: date = utils.parse_date(form_data['date']) except ValueError: template_dict['message'] = \ 'Date is not in the correct format (YYYY-MM-DD).' else: if key == 'new': try: article = NewsArticle(title=form_data['title'], author=form_data['author'], date=date, body=form_data['body']) article.put() except datastore_errors.Error: template_dict['message'] = \ 'Could not create new article.' else: template_dict['message'] = 'Article created.' template_dict['show_form'] = False else: try: article = NewsArticle.get(Key(key)) except BadKeyError: template_dict['message'] = \ 'Could not find article with key %r.' % key else: article.title = form_data['title'] article.author = form_data['author'] article.date = date article.body = form_data['body'] try: article.put() except datastore_errors.Error: template_dict['message'] = \ 'Could not save changes to article.' else: template_dict['form_data'] = article template_dict['message'] = 'Changes saved.' self.render_template('edit', template_dict)
def get(self): self.render_template('admin_news', {'news_list': NewsArticle.all().order('-date')})
def auto_article_go_getter(): print("starting builds ", file=sys.stderr) cnn_paper = newspaper.build("https://www.cnn.com", memorize_articles=True, language = 'en') print("cnn_paper built", file=sys.stderr) nbc_paper = newspaper.build("https://www.nbcnews.com", memorize_articles=True, language = 'en') #print("nbc_paper built", file=sys.stderr) #nyt_paper = newspaper.build("https://www.nytimes.com/", memorize_articles=True, language = 'en') #print("nyt_paper built", file=sys.stderr) apn_paper = newspaper.build("https://apnews.com/", memorize_articles=True, language = 'en') print("apn_paper built", file=sys.stderr) abc_paper = newspaper.build("https://abcnews.go.com/", memorize_articles=True, language = 'en') print("abc_paper built", file=sys.stderr) papers = [cnn_paper, nbc_paper, apn_paper, abc_paper] verge_paper = newspaper.build("https://www.theverge.com/", memorize_articles=True, language = 'en') print("verge_paper built", file=sys.stderr) techP = [verge_paper] espn_paper = newspaper.build("https://www.espn.com/", memorize_articles=True, language = 'en') print("espn_paper built", file=sys.stderr) sportP = [espn_paper] et_paper = newspaper.build("https://ew.com/", memorize_articles=True, language = 'en') print("ew_paper built", file=sys.stderr) entertainmentP = [et_paper] crypto_paper = newspaper.build("https://cryptonews.com/", memorize_articles=True, language = 'en') print("crypto_paper built", file=sys.stderr) cryptoP = [crypto_paper] climate_paper = newspaper.build("https://www.climatechangenews.com/", memorize_articles=True, language = 'en') print("climate_paper built", file=sys.stderr) climateP = [climate_paper] print("all papers built", file=sys.stderr) count = 0 article_list = [] print("Starting pool threading", file=sys.stderr) print("Starting pool for papers", file=sys.stderr) news_pool.set(papers, threads_per_source=1000) news_pool.join() print("Finished pool threading for papers", file=sys.stderr) print("Starting pool for techp", file=sys.stderr) news_pool.set(techP, threads_per_source=1000) news_pool.join() print("Finished pool threading for techp", file=sys.stderr) print("Starting pool for sportp", file=sys.stderr) news_pool.set(sportP, threads_per_source=1000) news_pool.join() print("Finished pool threading for sportp", file=sys.stderr) print("Starting pool for entertainmentp", file=sys.stderr) news_pool.set(entertainmentP, threads_per_source=1000) news_pool.join() print("Finished pool threading for entertainmentp", file=sys.stderr) print("Starting pool for cryptop", file=sys.stderr) news_pool.set(cryptoP, threads_per_source=1000) news_pool.join() print("Finished pool threading for cryptop", file=sys.stderr) print("Starting pool for climatep", file=sys.stderr) news_pool.set(climateP, threads_per_source=1000) news_pool.join() print("Finished pool threading for climatep", file=sys.stderr) print("Saving articles to mongodb", file=sys.stderr) for build in papers: for news in (build.articles): if "politics" in news.url and "cnnespanol" not in news.url: news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) if "apnews.com" in news.url: textSum = news.text article = NewsArticle( link = news.url, image = news.top_image, wing = "political", #text = news.text, text = textSum, title = news.title ).save() #email_services = ["hotmail", "gmail", "yahoo"] #email_contains_service = any(email_service in user_email for email_service in email_services) elif ["stock", "net", "loss", "Q1", "Q2", "Q3", "Q4", "Gain"] in word_tokenize(news.text): news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) if "apnews.com" in news.url: textSum = news.text article = NewsArticle( link = news.url, image = news.top_image, wing = "buisness", text = textSum, title = news.title ).save() elif "covid" in news.url or "corona" in news.url: news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) if "apnews.com" in news.url: textSum = news.text article = NewsArticle( link = news.url, image = news.top_image, wing = "covid", text = textSum, title = news.title ).save() count += 1 for build in techP: for news in (build.articles): news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) if "apnews.com" in news.url: textSum = news.text if "#comments" not in news.url: article = NewsArticle( link = news.url, image = news.top_image, wing = "tech", text = textSum, title = news.title ).save() for build in sportP: for news in (build.articles): news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) article = NewsArticle( link = news.url, image = news.top_image, wing = "sports", text = textSum, title = news.title ).save() for build in entertainmentP: for news in (build.articles): news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) article = NewsArticle( link = news.url, image = news.top_image, wing = "entertainment", text = textSum, title = news.title ).save() for build in cryptoP: for news in (build.articles): news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) article = NewsArticle( link = news.url, image = news.top_image, wing = "crypto", text = textSum, title = news.title ).save() for build in climateP: for news in (build.articles): news.parse() #call on text summarizer with text of article textSum = text_summarizer(news.text) article = NewsArticle( link = news.url, image = news.top_image, wing = "climate", text = textSum, title = news.title ).save() print("Articles saved in mongodb", file=sys.stderr)
def form(self, source, title, location, text, summary=None, keywords=[], open_graph=None, publish_date=None): article = NA.NewsArticle(source=source,location=location,title=title, text=text, summary=summary, keywords=keywords, open_graph=open_graph, publish_date=publish_date) return article
def newsarticle_index(): newarticles = NewsArticle.objects().order_by("-created") return jsonify([article.to_public_json() for article in newarticles])
def post(self, key): post = self.request.POST form_data = dict((k, post.get(k, '')) for k in ('title', 'author', 'date', 'body')) template_dict = {'form_data': form_data, 'key': key, 'show_form' : True} if 'delete_article' in post: try: NewsArticle.get(Key(key)).delete() except datastore_errors.Error: template_dict['message'] = \ 'Could not delete article with key %r.' % key else: template_dict['message'] = 'Article deleted.' template_dict['show_form'] = False else: try: date = utils.parse_date(form_data['date']) except ValueError: template_dict['message'] = \ 'Date is not in the correct format (YYYY-MM-DD).' else: if key == 'new': try: article = NewsArticle(title=form_data['title'], author=form_data['author'], date=date, body=form_data['body']) article.put() except datastore_errors.Error: template_dict['message'] = \ 'Could not create new article.' else: template_dict['message'] = 'Article created.' template_dict['show_form'] = False else: try: article = NewsArticle.get(Key(key)) except BadKeyError: template_dict['message'] = \ 'Could not find article with key %r.' % key else: article.title = form_data['title'] article.author = form_data['author'] article.date = date article.body = form_data['body'] try: article.put() except datastore_errors.Error: template_dict['message'] = \ 'Could not save changes to article.' else: template_dict['form_data'] = article template_dict['message'] = 'Changes saved.' self.render_template('edit', template_dict)
def newsarticle_index(id: str): if id == "political": count = 0 artic = [] for article in NewsArticle.objects(): if (article.wing == "political") and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "stocks": count = 0 artic = [] for article in NewsArticle.objects(): if (article.wing == "buisness") and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "covid": artic = [] count = 0 for article in NewsArticle.objects(): if (article.wing == "covid") and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "everything": artic = [] count = 0 for article in NewsArticle.objects(): if (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 else: break return jsonify([article.to_public_json() for article in artic]) elif id == "tech": artic = [] count = 0 for article in NewsArticle.objects(): if (article.wing == "tech" and "#comments" not in article.link) and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "sports": artic = [] count = 0 for article in NewsArticle.objects(): if (article.wing == "sports") and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "entertainment": artic = [] count = 0 for article in NewsArticle.objects(): if (article.wing == "entertainment" and "magazine.store" not in article.link and "https://ew.com" in article.link) and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "crypto": artic = [] count = 0 for article in NewsArticle.objects(): if (article.wing == "crypto" and ".htm" in article.link) and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) elif id == "climate": artic = [] count = 0 for article in NewsArticle.objects(): if (article.wing == "climate") and (count < 10): print(article.link, file=sys.stderr) artic.insert(0, article) count + 1 return jsonify([article.to_public_json() for article in artic]) else: return "error"