def post(self):
post = self.request.POST
if post['kind'] == 'badge':
badge = Badge(name=post['name'],
description=post['description'],
category=post['category'],
image=post['image'],
value=int(post['value']))
badge.save()
elif post['kind'] == 'article':
date = datetime.datetime.strptime(post['date'], '%Y-%m-%d').date()
article = NewsArticle(title=post['title'],
author=post['author'],
body=post['body'],
date=date)
article.save()
elif post['kind'] == 'award':
badge = Badge.get_by_id(int(post['badge']))
for member in post.getall('members'):
member = Member.get_by_id(int(member))
award = Award(member=member,
badge=badge,
date=datetime.date.today(),
proof=post['proof'])
award.save()
elif post['kind'] == 'talk':
talk = Talk(title=post['title'],
date=datetime.datetime.strptime(
post['date'], '%Y-%m-%d').date(),
description=post['description'],
member=Member.get_by_id(int(post['member'])),
video=post['video'])
talk.put()
self.get()
def get_gap_fill_questions():
url = request.args.get('url')
limit = int(request.args.get('limit') or 50)
article_text = get_article_text(url)
questions = generate_gap_fill_questions(article_text)
num_questions = len(questions)
for key in ["_type", "_subtype"]:
for q in questions:
q.pop(key)
try:
db.session.add(
QuestionGenRequest(url=url,
questions=questions,
question_type="gap_fill"))
db.session.commit()
except Exception as e:
print(e)
try:
parsed_url = urlparse(url)
article_id = md5(article_text.strip().encode('utf-8'))
if not NewsArticle.query.get(article_id.hexdigest()):
db.session.add(
NewsArticle(id=article_id.hexdigest(),
url=url,
article_text=article_text.strip(),
domain=parsed_url.netloc))
db.session.commit()
except Exception as e:
print(e)
for q in questions:
q_id = md5(q.get('question').encode('utf-8'))
if not Question.query.get(q_id.hexdigest()):
try:
db.session.add(
Question(id=q_id.hexdigest(),
question_text=q.get("question"),
source_sentence=q.get("source_sentence"),
correct_answer=q.get("answer"),
good_question_votes=0,
bad_question_votes=0,
news_article_id=article_id.hexdigest()))
db.session.commit()
except Exception as e:
print("Unable to add item to database.")
print(e)
news_article = NewsArticle.query.get(article_id.hexdigest())
questions = question_schema.dump(news_article.questions.all()).data
shuffle(questions)
return jsonify({
'num_questions': min(limit, num_questions),
'questions': questions[:limit]
})
def newsarticle_create(username: str):
article = NewsArticle(
link = "test link",
wing = "test wing",
text = "some sample text"
).save()
return article
def post(self):
post = self.request.POST
if "delete_article" in post:
output = post.getall('delete_article')
try:
counter = 0
for articleKey in output:
article = NewsArticle.get(Key(articleKey))
article.delete()
counter += 1
message = '%d Article(s) deleted.' % counter
except datastore_errors.Error:
message = 'Database error, a subset of your selection might have been deleted'
else:
message = 'No articles selected, congratulations, you wasted some CPU time.'
news_list = NewsArticle.all().order('-date');
self.render_template('admin_news', { 'news_list' : news_list, 'delete_successful' : message })
def post(self):
articles_deleted = 0
for article_key in self.request.POST.getall('delete_article'):
try:
article_key = Key(article_key)
except BadKeyError:
# Wrong syntax for a key, move on to next key.
continue
article = NewsArticle.get(article_key)
if article:
article.delete()
articles_deleted += 1
# Else, not article has this key.
self.render_template('admin_news',
{'news_list': NewsArticle.all().order('-date'),
'delete_successful': '%d article(s) deleted.' % articles_deleted})
def post(self):
post = self.request.POST
if post['kind'] == 'badge':
b = Badge(name=post['name'],
description=post['description'],
category=post['category'],
image=post['image'],
value=int(post['value']))
b.save()
elif post['kind'] == 'article':
a = NewsArticle(title=post['title'],
author=post['author'],
body=post['body'],
date=datetime.date.today())
a.save()
elif post['kind'] == 'award':
badge = Badge.gql('WHERE name = :1', post['badge']).get()
for h in post.getall('hackers'):
hacker = Hacker.gql('WHERE handle = :1', h).get()
a = Award(hacker=hacker,
badge=badge,
date=datetime.date.today(),
proof=post['proof'])
a.save()
hacker.score_cache = hacker.score + badge.value
hacker.save()
self.get()
def post(self):
post = self.request.POST
if post['kind'] == 'badge':
badge = Badge(
name=post['name'],
description=post['description'],
category=post['category'],
image=post['image'],
value=int(post['value'])
)
badge.save()
elif post['kind'] == 'article':
date = datetime.datetime.strptime(post['date'], '%Y-%m-%d').date()
article = NewsArticle(
title=post['title'],
author=post['author'],
body=post['body'],
date=date
)
article.save()
elif post['kind'] == 'award':
badge = Badge.get_by_id(int(post['badge']))
for member in post.getall('members'):
member = Member.get_by_id(int(member))
award = Award(
member=member,
badge=badge,
date=datetime.date.today(),
proof=post['proof']
)
award.save()
member.score_cache = member.score + badge.value
member.save()
elif post['kind'] == 'talk':
talk = Talk(
title=post['title'],
date=datetime.datetime.strptime(post['date'], '%Y-%m-%d').date(),
description=post['description'],
member=Member.get_by_id(int(post['member'])),
video=post['video']
)
talk.put()
self.get()
def get_rows_to_create(json_data_from_file: Dict) -> List[NewsArticle]:
rows_to_add = list()
len_of_rows = len(json_data_from_file)
print(f"Total rows to import {len_of_rows}")
for index, json_data in enumerate(json_data_from_file):
print(f"Import {index} of {len_of_rows}")
_preprocess_data(json_data)
new_row = NewsArticle(**json_data)
rows_to_add.append(new_row)
return rows_to_add
def post(self):
post = self.request.POST
if "delete_article" in post:
output = post.getall('delete_article')
try:
counter = 0
for articleKey in output:
article = NewsArticle.get(Key(articleKey))
article.delete()
counter += 1
message = '%d Article(s) deleted.' % counter
except datastore_errors.Error:
message = 'Database error, a subset of your selection might have been deleted'
else:
message = 'No articles selected, congratulations, you wasted some CPU time.'
news_list = NewsArticle.all().order('-date')
self.render_template('admin_news', {
'news_list': news_list,
'delete_successful': message
})
def get(self, key):
template_dict = {'key': key, 'show_form' : True}
if key == 'new':
template_dict['form_data'] = {
'title': '',
'author': Member.get_current_member().handle,
'date': datetime.date.today(),
'body': ''}
else:
try:
template_dict['form_data'] = NewsArticle.get(Key(key))
except BadKeyError:
template_dict['message'] = \
'Could not find article with key %r.' % key
template_dict['show_form'] = False
self.render_template('edit', template_dict)
def get(self, key):
template_dict = {'key': key, 'show_form': True}
if key == 'new':
template_dict['form_data'] = {
'title': '',
'author': Member.get_current_member().handle,
'date': datetime.date.today(),
'body': ''
}
else:
try:
template_dict['form_data'] = NewsArticle.get(Key(key))
except BadKeyError:
template_dict['message'] = \
'Could not find article with key %r.' % key
template_dict['show_form'] = False
self.render_template('edit', template_dict)
def addNews():
data = request.json
news = News()
for article in data['articles']:
newArticle = NewsArticle(title=article['title'],
description=article['description'],
lang=article['lang'])
news.articles.append(newArticle)
db.session.add(news)
db.session.commit()
return Response(json.dumps(
{'message': 'News has been created succesfully'}),
status=200,
mimetype='application/json')
def post(self):
post = self.request.POST
if post['kind'] == 'badge':
b = Badge(name=post['name'], description=post['description'],
category=post['category'], image=post['image'],
value=int(post['value']))
b.save()
elif post['kind'] == 'article':
a = NewsArticle(title=post['title'], author=post['author'],
body=post['body'], date=datetime.date.today())
a.save()
elif post['kind'] == 'award':
badge = Badge.gql('WHERE name = :1', post['badge']).get()
for h in post.getall('hackers'):
hacker = Hacker.gql('WHERE handle = :1', h).get()
a=Award(hacker=hacker,
badge=badge,
date=datetime.date.today(),
proof=post['proof'])
a.save()
hacker.score_cache = hacker.score + badge.value
hacker.save()
self.get()
def get(self):
self.render_template('admin_news',
{'news_list' : NewsArticle.all().order('-date')})
def post(self, key):
post = self.request.POST
form_data = dict(
(k, post.get(k, '')) for k in ('title', 'author', 'date', 'body'))
template_dict = {'form_data': form_data, 'key': key, 'show_form': True}
if 'delete_article' in post:
try:
NewsArticle.get(Key(key)).delete()
except datastore_errors.Error:
template_dict['message'] = \
'Could not delete article with key %r.' % key
else:
template_dict['message'] = 'Article deleted.'
template_dict['show_form'] = False
else:
try:
date = utils.parse_date(form_data['date'])
except ValueError:
template_dict['message'] = \
'Date is not in the correct format (YYYY-MM-DD).'
else:
if key == 'new':
try:
article = NewsArticle(title=form_data['title'],
author=form_data['author'],
date=date,
body=form_data['body'])
article.put()
except datastore_errors.Error:
template_dict['message'] = \
'Could not create new article.'
else:
template_dict['message'] = 'Article created.'
template_dict['show_form'] = False
else:
try:
article = NewsArticle.get(Key(key))
except BadKeyError:
template_dict['message'] = \
'Could not find article with key %r.' % key
else:
article.title = form_data['title']
article.author = form_data['author']
article.date = date
article.body = form_data['body']
try:
article.put()
except datastore_errors.Error:
template_dict['message'] = \
'Could not save changes to article.'
else:
template_dict['form_data'] = article
template_dict['message'] = 'Changes saved.'
self.render_template('edit', template_dict)
def get(self):
self.render_template('admin_news',
{'news_list': NewsArticle.all().order('-date')})
def auto_article_go_getter():
print("starting builds ", file=sys.stderr)
cnn_paper = newspaper.build("https://www.cnn.com", memorize_articles=True, language = 'en')
print("cnn_paper built", file=sys.stderr)
nbc_paper = newspaper.build("https://www.nbcnews.com", memorize_articles=True, language = 'en')
#print("nbc_paper built", file=sys.stderr)
#nyt_paper = newspaper.build("https://www.nytimes.com/", memorize_articles=True, language = 'en')
#print("nyt_paper built", file=sys.stderr)
apn_paper = newspaper.build("https://apnews.com/", memorize_articles=True, language = 'en')
print("apn_paper built", file=sys.stderr)
abc_paper = newspaper.build("https://abcnews.go.com/", memorize_articles=True, language = 'en')
print("abc_paper built", file=sys.stderr)
papers = [cnn_paper, nbc_paper, apn_paper, abc_paper]
verge_paper = newspaper.build("https://www.theverge.com/", memorize_articles=True, language = 'en')
print("verge_paper built", file=sys.stderr)
techP = [verge_paper]
espn_paper = newspaper.build("https://www.espn.com/", memorize_articles=True, language = 'en')
print("espn_paper built", file=sys.stderr)
sportP = [espn_paper]
et_paper = newspaper.build("https://ew.com/", memorize_articles=True, language = 'en')
print("ew_paper built", file=sys.stderr)
entertainmentP = [et_paper]
crypto_paper = newspaper.build("https://cryptonews.com/", memorize_articles=True, language = 'en')
print("crypto_paper built", file=sys.stderr)
cryptoP = [crypto_paper]
climate_paper = newspaper.build("https://www.climatechangenews.com/", memorize_articles=True, language = 'en')
print("climate_paper built", file=sys.stderr)
climateP = [climate_paper]
print("all papers built", file=sys.stderr)
count = 0
article_list = []
print("Starting pool threading", file=sys.stderr)
print("Starting pool for papers", file=sys.stderr)
news_pool.set(papers, threads_per_source=1000)
news_pool.join()
print("Finished pool threading for papers", file=sys.stderr)
print("Starting pool for techp", file=sys.stderr)
news_pool.set(techP, threads_per_source=1000)
news_pool.join()
print("Finished pool threading for techp", file=sys.stderr)
print("Starting pool for sportp", file=sys.stderr)
news_pool.set(sportP, threads_per_source=1000)
news_pool.join()
print("Finished pool threading for sportp", file=sys.stderr)
print("Starting pool for entertainmentp", file=sys.stderr)
news_pool.set(entertainmentP, threads_per_source=1000)
news_pool.join()
print("Finished pool threading for entertainmentp", file=sys.stderr)
print("Starting pool for cryptop", file=sys.stderr)
news_pool.set(cryptoP, threads_per_source=1000)
news_pool.join()
print("Finished pool threading for cryptop", file=sys.stderr)
print("Starting pool for climatep", file=sys.stderr)
news_pool.set(climateP, threads_per_source=1000)
news_pool.join()
print("Finished pool threading for climatep", file=sys.stderr)
print("Saving articles to mongodb", file=sys.stderr)
for build in papers:
for news in (build.articles):
if "politics" in news.url and "cnnespanol" not in news.url:
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
if "apnews.com" in news.url:
textSum = news.text
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "political",
#text = news.text,
text = textSum,
title = news.title
).save()
#email_services = ["hotmail", "gmail", "yahoo"]
#email_contains_service = any(email_service in user_email for email_service in email_services)
elif ["stock", "net", "loss", "Q1", "Q2", "Q3", "Q4", "Gain"] in word_tokenize(news.text):
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
if "apnews.com" in news.url:
textSum = news.text
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "buisness",
text = textSum,
title = news.title
).save()
elif "covid" in news.url or "corona" in news.url:
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
if "apnews.com" in news.url:
textSum = news.text
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "covid",
text = textSum,
title = news.title
).save()
count += 1
for build in techP:
for news in (build.articles):
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
if "apnews.com" in news.url:
textSum = news.text
if "#comments" not in news.url:
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "tech",
text = textSum,
title = news.title
).save()
for build in sportP:
for news in (build.articles):
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "sports",
text = textSum,
title = news.title
).save()
for build in entertainmentP:
for news in (build.articles):
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "entertainment",
text = textSum,
title = news.title
).save()
for build in cryptoP:
for news in (build.articles):
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "crypto",
text = textSum,
title = news.title
).save()
for build in climateP:
for news in (build.articles):
news.parse()
#call on text summarizer with text of article
textSum = text_summarizer(news.text)
article = NewsArticle(
link = news.url,
image = news.top_image,
wing = "climate",
text = textSum,
title = news.title
).save()
print("Articles saved in mongodb", file=sys.stderr)
def form(self, source, title, location, text, summary=None, keywords=[], open_graph=None,
publish_date=None):
article = NA.NewsArticle(source=source,location=location,title=title, text=text,
summary=summary, keywords=keywords, open_graph=open_graph,
publish_date=publish_date)
return article
def newsarticle_index():
newarticles = NewsArticle.objects().order_by("-created")
return jsonify([article.to_public_json() for article in newarticles])
def post(self, key):
post = self.request.POST
form_data = dict((k, post.get(k, ''))
for k in ('title', 'author', 'date', 'body'))
template_dict = {'form_data': form_data, 'key': key, 'show_form' : True}
if 'delete_article' in post:
try:
NewsArticle.get(Key(key)).delete()
except datastore_errors.Error:
template_dict['message'] = \
'Could not delete article with key %r.' % key
else:
template_dict['message'] = 'Article deleted.'
template_dict['show_form'] = False
else:
try:
date = utils.parse_date(form_data['date'])
except ValueError:
template_dict['message'] = \
'Date is not in the correct format (YYYY-MM-DD).'
else:
if key == 'new':
try:
article = NewsArticle(title=form_data['title'],
author=form_data['author'],
date=date,
body=form_data['body'])
article.put()
except datastore_errors.Error:
template_dict['message'] = \
'Could not create new article.'
else:
template_dict['message'] = 'Article created.'
template_dict['show_form'] = False
else:
try:
article = NewsArticle.get(Key(key))
except BadKeyError:
template_dict['message'] = \
'Could not find article with key %r.' % key
else:
article.title = form_data['title']
article.author = form_data['author']
article.date = date
article.body = form_data['body']
try:
article.put()
except datastore_errors.Error:
template_dict['message'] = \
'Could not save changes to article.'
else:
template_dict['form_data'] = article
template_dict['message'] = 'Changes saved.'
self.render_template('edit', template_dict)
def newsarticle_index(id: str):
if id == "political":
count = 0
artic = []
for article in NewsArticle.objects():
if (article.wing == "political") and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "stocks":
count = 0
artic = []
for article in NewsArticle.objects():
if (article.wing == "buisness") and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "covid":
artic = []
count = 0
for article in NewsArticle.objects():
if (article.wing == "covid") and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "everything":
artic = []
count = 0
for article in NewsArticle.objects():
if (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
else:
break
return jsonify([article.to_public_json() for article in artic])
elif id == "tech":
artic = []
count = 0
for article in NewsArticle.objects():
if (article.wing == "tech"
and "#comments" not in article.link) and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "sports":
artic = []
count = 0
for article in NewsArticle.objects():
if (article.wing == "sports") and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "entertainment":
artic = []
count = 0
for article in NewsArticle.objects():
if (article.wing == "entertainment"
and "magazine.store" not in article.link
and "https://ew.com" in article.link) and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "crypto":
artic = []
count = 0
for article in NewsArticle.objects():
if (article.wing == "crypto"
and ".htm" in article.link) and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
elif id == "climate":
artic = []
count = 0
for article in NewsArticle.objects():
if (article.wing == "climate") and (count < 10):
print(article.link, file=sys.stderr)
artic.insert(0, article)
count + 1
return jsonify([article.to_public_json() for article in artic])
else:
return "error"