def update_articles_in_db():
feed = feedparser.parse(FEED_URL)
counter = 0
db_articles = Article.query.all()
db_articles_ids = [article.article_id for article in db_articles]
for i in feed["entries"]:
if int(i["id"]) not in db_articles_ids:
published_date = datetime.strptime(i["published"],
"%a, %d %b %Y %H:%M:%S %z")
new_article = Article(
article_id=i["id"],
published_date=published_date,
title=i["title"],
creator=i["author"],
image_src=i["szn_image"],
link_src=i["link"],
text=i["summary"],
keywords="TODO",
media_name="cti-doma",
)
db.session.add(new_article)
counter = counter + 1
db.session.commit()
return counter
def setUp(self):
'''
Set up method that will run before every test
'''
self.new_article = Article('abcnews', 'larry', 'drones today',
'drones replacing armies',
'https:abcnew.com', 'null',
'2019-11-18T01:51:09Z')
def get_articles_by_story(self, story_id):
story=Story.find_by_id(story_id)
if story==None:
s=relegence_API.stories.by_story_id(story_id, {'numDocs': 100})
story=smodel=Story(story_id=s['id'], title=s['title'], mag_score=s['magScore'],
num_total_docs=s['numTotalDocs'], num_original_docs=s['numOriginalDocs']
# ,topic_creation_time= s['topicCreationTime'], last_topic_activity_time = s['lastTopicActivityTime']
)
smodel.save()
articles=s['articles']
for a in articles:
fname=a['id'].replace('/','|')
amodel=Article(article_id=fname, story=smodel, source=a['source']['title'], \
link=a['link'], title=a['title'], snippet=a['snippet'], \
author=a['author']['name'], text=a['content'], file_name=fname, \
source_id=a['source']['id'])
amodel.save()
return Article.find_by_story(story)
def setUp(self):
'''
Set up method that will run before every Test
'''
self.new_article = Article('Kevin Kittony', 'Big change in Kenya',
'get the tips to making big changes',
'https://newsapi.org/v2/image',
'2019-04-12,',
'https://newsapi.org/v2/bigchanges')
def post_article() -> Dict[str, str]:
url = request.form.get("url")
article = Article(url=url, date_added=int(today()))
db.session.add(article)
db.session.commit()
id = article.id
articles = Article.query.all()
data = {x.id: serialize_article(x) for x in articles}
return data[id]
def _make_article(self):
for i in range(self.article_num):
title = self.blog_fake.text()[:8]
content = self.blog_fake.text()
abstract = self.blog_fake.text()[:16]
like_num = self.blog_fake.random_int()
view_num = self.blog_fake.random_int()
post_time = self.blog_fake.date_time()
# print(title)
post_uuid = Article.generate_uuid(title)
new_post = Article(
title=title,
content=content,
abstract=abstract,
like_num=like_num,
post_time=post_time,
uuid=post_uuid,
view_num=view_num,
img_url="https://s1.ax1x.com/2020/03/24/8bg3oF.png")
db.session.add(new_post)
self.articles.append(new_post)
def update_article(self, article, data):
"""
修改文章
:param article: 数据库查询的article
:param data: 需要更新的数据
:return: None
"""
# 处理原文章没有标签添加标签时出错的问题
if article.tags.all() is None:
origin_tag_name_list = None
else:
origin_tag_name_list = [tag.name for tag in article.tags.all()]
# 处理原文章没有分类时添加分类出错的问题
if article.category is None:
origin_category_name = None
else:
origin_category_name = article.category.name
title, content, abstract, like_num, c_name, t_names, view_num, post_time, img_url = get_values_by_keys(
data, [('title', article.title), ('content', article.content),
('abstract', article.abstract),
('likeNum', article.like_num),
('category', origin_category_name),
('tags', origin_tag_name_list),
('viewNum', article.view_num),
('postTime', article.post_time),
("coverImgUrl", article.img_url)])
try:
post_uuid = article.uuid
if title is not article.title: # title值被修改需要重新生成uuid
post_uuid = ArticleModel.generate_uuid(title)
article.uuid = post_uuid
article.title = title
article.content = content
article.abstract = abstract
article.like_num = like_num
article.category = self.requester.get_model_by_name(
CategoryModel, c_name)
article.tags = self.requester.get_model_by_names(TagModel, t_names)
article.view_num = view_num
article.img_url = img_url
# 更新发布时间时需要将前端传来的时间戳转化为datetime对象
if type(post_time) is int:
post_time = get_datetime(int(post_time))
article.post_time = post_time
self.requester.commit()
return True
except:
return False
def post(self):
data = request.form
form = ArticleForm(data)
if form.validate():
new_article = Article(title=data['title'],
content=data['content'],
post_date=data['post_date'])
db.session.add(new_article)
db.session.commit()
return response({'id': new_article.id})
else:
return response(form.get_error_messages(), 400)
def build_article(self,
title,
content,
abstract,
c_name,
t_names,
img_url="",
post_time=None,
like_num=0,
view_num=0):
"""
构建文章
:param img_url:
:param abstract:
:param view_num:
:param post_time:
:param like_num:
:param title: 标题
:param content: 内容
:param c_name: 分类名
:param t_names: 标签名
:return: 状态码，new post
"""
if post_time is None:
post_time = get_time()
else:
post_time = get_datetime(int(post_time))
try:
category = self.requester.get_model_by_name(CategoryModel,
name=c_name)
tags = self.requester.get_model_by_names(TagModel, names=t_names)
# 添加uuid
uuid = ArticleModel.generate_uuid(title)
new_post = ArticleModel(title=title,
content=content,
abstract=abstract,
like_num=like_num,
post_time=post_time,
category=category,
tags=tags,
uuid=uuid,
view_num=view_num,
img_url=img_url)
except:
return False, None
return True, new_post
import app.model.ArticleTitleList as al
import app.model.Article as article
import jieba
import jieba.analyse
url = "http://tech.qq.com/l/201804/scroll_20.htm"
qqTitleList = al.ArticelTitleList(url)
dateStr = qqTitleList.findDateFromUrl()
print(dateStr)
titleInfoList = qqTitleList.getAllTitles()
articleInfoList = []
for titleInfo in titleInfoList:
##print(titleInfo.id)
##print(titleInfo.title)
qqArticle = article.QqArticel(titleInfo.url)
try:
qqArticle._soup2Data()
except Exception:
qqArticle = None
##print(qqArticle.data)
if qqArticle is not None:
articleInfoList.append(qqArticle)
##titleKeywords = jieba.analyse.extract_tags(qqArticle.data['title'], topK=3, withWeight=False, allowPOS=('ns', 'n', 'vn', 'v'))
##print(titleKeywords)
keywords = jieba.analyse.extract_tags(qqArticle.data['context'],
topK=3,
withWeight=False,
allowPOS=('ns', 'n', 'vn', 'v'))
print(qqArticle.data["title"])
##print(keywords)
def get(self, article_id):
a=Article.by_id(article_id)
return a.to_mongo()