def make_topic(ref, title, desc): """ Добавляет новую тему в таблицу :param ref: ссылка :param title: название :param desc: описание """ print('new topic') all_topic_text = '' topic_words_len = defaultdict(int) topic_words_freq = defaultdict(int) articles = Parser(ref) times_articles = articles.get_time() a_titles, a_description, a_refs = articles.get_titles() for j in range(len(a_titles)): print('new article') article_words_len = defaultdict(int) article_words_freq = defaultdict(int) article = Parser(a_refs[j]) all_article_text = article.get_paragraphs() all_topic_text += ' ' + all_article_text fill_words(all_article_text.split(), article_words_freq, article_words_len) new_article = Article(topic=title, name=a_titles[j], href=a_refs[j], text=article.get_paragraphs(), upd=dateparser.parse(times_articles[j].text), stat_words_len=json.dumps(article_words_len), stat_words_freq=json.dumps(article_words_freq)) new_article.save() make_tags(article.get_tags(), a_titles[j]) fill_words(all_topic_text.split(), topic_words_freq, topic_words_len) new_topic = Topic(name=title, description=desc, href=ref, upd=dateparser.parse(times_articles[0].text), stat_words_len=json.dumps(topic_words_len), stat_words_freq=json.dumps(topic_words_freq)) new_topic.save()
for j in range(len(times_articles)): if dateparser.parse(times_articles[j].text) > last_upd: have_new = True print('new article') article = Parser(a_refs[j]) article_words_len = defaultdict(int) article_words_freq = defaultdict(int) all_article_text = article.get_paragraphs() fill_words(all_article_text.split(), article_words_freq, article_words_len) new_article = Article(topic=titles[index], name=a_titles[j], href=a_refs[j], text=article.get_paragraphs(), upd=dateparser.parse(times_articles[j].text), stat_words_len=json.dumps(article_words_len), stat_words_freq=json.dumps(article_words_freq)) new_article.save() make_tags(article.get_tags(), a_titles[j]) else: break if have_new: updated_articles = Article.select()\ .where(Article.topic == cur_topic.name) cur_topic.stat_words_len, cur_topic.stat_words_freq = update_topic_stat(cur_topic, updated_articles) cur_topic.save() db.close() print('done') sleep(config.TIME_TO_SLEEP) except: print('ups')