def make_topic(ref, title, desc):
"""
Добавляет новую тему в таблицу
:param ref: ссылка
:param title: название
:param desc: описание
"""
print('new topic')
all_topic_text = ''
topic_words_len = defaultdict(int)
topic_words_freq = defaultdict(int)
articles = Parser(ref)
times_articles = articles.get_time()
a_titles, a_description, a_refs = articles.get_titles()
for j in range(len(a_titles)):
print('new article')
article_words_len = defaultdict(int)
article_words_freq = defaultdict(int)
article = Parser(a_refs[j])
all_article_text = article.get_paragraphs()
all_topic_text += ' ' + all_article_text
fill_words(all_article_text.split(), article_words_freq, article_words_len)
new_article = Article(topic=title, name=a_titles[j],
href=a_refs[j],
text=article.get_paragraphs(),
upd=dateparser.parse(times_articles[j].text),
stat_words_len=json.dumps(article_words_len),
stat_words_freq=json.dumps(article_words_freq))
new_article.save()
make_tags(article.get_tags(), a_titles[j])
fill_words(all_topic_text.split(), topic_words_freq, topic_words_len)
new_topic = Topic(name=title, description=desc, href=ref,
upd=dateparser.parse(times_articles[0].text),
stat_words_len=json.dumps(topic_words_len),
stat_words_freq=json.dumps(topic_words_freq))
new_topic.save()
for j in range(len(times_articles)):
if dateparser.parse(times_articles[j].text) > last_upd:
have_new = True
print('new article')
article = Parser(a_refs[j])
article_words_len = defaultdict(int)
article_words_freq = defaultdict(int)
all_article_text = article.get_paragraphs()
fill_words(all_article_text.split(), article_words_freq, article_words_len)
new_article = Article(topic=titles[index], name=a_titles[j],
href=a_refs[j],
text=article.get_paragraphs(),
upd=dateparser.parse(times_articles[j].text),
stat_words_len=json.dumps(article_words_len),
stat_words_freq=json.dumps(article_words_freq))
new_article.save()
make_tags(article.get_tags(), a_titles[j])
else:
break
if have_new:
updated_articles = Article.select()\
.where(Article.topic == cur_topic.name)
cur_topic.stat_words_len, cur_topic.stat_words_freq = update_topic_stat(cur_topic, updated_articles)
cur_topic.save()
db.close()
print('done')
sleep(config.TIME_TO_SLEEP)
except:
print('ups')