def populate_db(): db_create() ucontr = UserController() ccontr = CategoryController() fcontr = FeedController() acontr = ArticleController() ccontr = CategoryController() user1, user2 = [ ucontr.create(login=name, email="*****@*****.**" % name, password=name) for name in ["user1", "user2"] ] article_total = 0 for user in (user1, user2): for i in range(3): cat_id = None if i: cat_id = ccontr.create(user_id=user.id, name="category%d" % i).id feed = fcontr.create(link="feed%d" % i, user_id=user.id, category_id=cat_id, title="%s feed%d" % (user.login, i)) for j in range(3): entry = "%s %s article%d" % (user.login, feed.title, j) article_total += 1 acontr.create(entry_id=entry, link='http://test.te/%d' % article_total, feed_id=feed.id, user_id=user.id, category_id=cat_id, title=entry, content="content %d" % article_total)
def populate_db(): db_create() ucontr = UserController() ccontr = CategoryController() fcontr = FeedController() acontr = ArticleController() ccontr = CategoryController() user1, user2 = [ucontr.create(login=name, email="*****@*****.**" % name, password=name) for name in ["user1", "user2"]] article_total = 0 for user in (user1, user2): for i in range(3): cat_id = None if i: cat_id = ccontr.create(user_id=user.id, name="category%d" % i).id feed = fcontr.create(link="feed%d" % i, user_id=user.id, category_id=cat_id, title="%s feed%d" % (user.login, i)) for j in range(3): entry = "%s %s article%d" % (user.login, feed.title, j) article_total += 1 acontr.create(entry_id=entry, link='http://test.te/%d' % article_total, feed_id=feed.id, user_id=user.id, category_id=cat_id, title=entry, content="content %d" % article_total)
async def insert_articles(queue, nḅ_producers=1): """Consumer coroutines. """ nb_producers_done = 0 while True: item = await queue.get() if item is None: nb_producers_done += 1 if nb_producers_done == nḅ_producers: print('All producers done.') print('Process finished.') break continue user, feed, articles = item if None is articles: logger.info('None') articles = [] logger.info('Inserting articles for {}'.format(feed.link)) art_contr = ArticleController(user.id) for article in articles: new_article = await construct_article(article, feed) try: existing_article_req = art_contr.read( user_id=user.id, feed_id=feed.id, entry_id=extract_id(article)) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 if exist: continue # insertion of the new article try: art_contr.create(**new_article) logger.info('New article added: {}'.format(new_article['link'])) except Exception: logger.exception('Error when inserting article in database.') continue
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.debug('inserting articles for {}'.format(feed.title)) logger.info("Database insertion...") new_articles = [] art_contr = ArticleController(user.id) for article in articles: existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) exist = existing_article_req.count() != 0 if exist: # if the article has been already retrieved, we only update # the content or the title logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) existing_article = existing_article_req.first() new_updated_date = None try: new_updated_date = dateutil.parser.parse(article['updated']) except Exception as e: print(e) #new_updated_date = existing_article.date if None is existing_article.updated_date: existing_article.updated_date = new_updated_date.replace( tzinfo=None) if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \ new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'): existing_article.updated_date = \ new_updated_date.replace(tzinfo=None) if existing_article.title != article['title']: existing_article.title = article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue article = construct_article(article, feed) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) except Exception: logger.exception("Error when inserting article in database:") continue return new_articles
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.info('Inserting articles for {}'.format(feed.title)) new_articles = [] art_contr = ArticleController(user.id) for article in articles: new_article = await construct_article(article, feed) try: existing_article_req = art_contr.read(feed_id=feed.id, entry_id=extract_id(article)) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 if exist: continue # if the article has been already retrieved, we only update # the content or the title logger.info('Article already in the database: {}'. \ format(article['link'])) existing_article = existing_article_req.first() if new_article['date'].replace(tzinfo=None) != \ existing_article.date: existing_article.date = new_article['date'] existing_article.updated_date = new_article['date'] if existing_article.title != new_article['title']: existing_article.title = new_article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) logger.info('Article updated: {}'.format(article['link'])) continue # insertion of the new article try: new_articles.append(art_contr.create(**new_article)) logger.info('New article added: {}'.format(new_article['link'])) except Exception: logger.exception('Error when inserting article in database:') continue return new_articles
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.debug('inserting articles for {}'.format(feed.title)) logger.info("Database insertion...") new_articles = [] art_contr = ArticleController(user.id) for article in articles: existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) exist = existing_article_req.count() != 0 if exist: # if the article has been already retrieved, we only update # the content or the title logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) existing_article = existing_article_req.first() new_updated_date = None try: new_updated_date = dateutil.parser.parse(article['updated']) except Exception as e: print(e)#new_updated_date = existing_article.date if None is existing_article.updated_date: existing_article.updated_date = new_updated_date.replace(tzinfo=None) if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \ new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'): existing_article.updated_date = \ new_updated_date.replace(tzinfo=None) if existing_article.title != article['title']: existing_article.title = article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue article = construct_article(article, feed) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) except Exception: logger.exception("Error when inserting article in database:") continue return new_articles
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.info('Inserting articles for {}'.format(feed.title)) new_articles = [] art_contr = ArticleController(user.id) for article in articles: new_article = await construct_article(article, feed) try: existing_article_req = art_contr.read(feed_id=feed.id, entry_id=extract_id(article)) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 if exist: # if the article has been already retrieved, we only update # the content or the title logger.info('Article already in the database: {}'. \ format(article['link'])) existing_article = existing_article_req.first() if new_article['date'].replace(tzinfo=None) != \ existing_article.date: existing_article.date = new_article['date'] existing_article.updated_date = new_article['date'] if existing_article.title != new_article['title']: existing_article.title = new_article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) logger.info('Article updated: {}'.format(article['link'])) continue # insertion of the new article try: new_articles.append(art_contr.create(**new_article)) logger.info('New article added: {}'.format(new_article['link'])) except Exception: logger.exception('Error when inserting article in database:') continue return new_articles
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.debug('inserting articles for {}'.format(feed.title)) logger.info("Database insertion...") new_articles = [] art_contr = ArticleController(user.id) for article in articles: existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) exist = existing_article_req.count() != 0 if exist: existing_article = existing_article_req.first() is_updated = False logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) content = get_article_content(article) if existing_article.title != article['title']: existing_article.title = article['title'] is_updated = True if existing_article.content != content: existing_article.content = content existing_article.readed = False is_updated = True if is_updated: art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue article = construct_article(article, feed.dump()) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) except Exception: logger.exception("Error when inserting article in database:") continue return new_articles