Exemplo n.º 1
0
def populate_db():
    db_create()
    ucontr = UserController()
    ccontr = CategoryController()
    fcontr = FeedController()
    acontr = ArticleController()
    ccontr = CategoryController()
    user1, user2 = [
        ucontr.create(login=name, email="*****@*****.**" % name, password=name)
        for name in ["user1", "user2"]
    ]

    article_total = 0
    for user in (user1, user2):
        for i in range(3):
            cat_id = None
            if i:
                cat_id = ccontr.create(user_id=user.id,
                                       name="category%d" % i).id
            feed = fcontr.create(link="feed%d" % i,
                                 user_id=user.id,
                                 category_id=cat_id,
                                 title="%s feed%d" % (user.login, i))
            for j in range(3):
                entry = "%s %s article%d" % (user.login, feed.title, j)
                article_total += 1
                acontr.create(entry_id=entry,
                              link='http://test.te/%d' % article_total,
                              feed_id=feed.id,
                              user_id=user.id,
                              category_id=cat_id,
                              title=entry,
                              content="content %d" % article_total)
Exemplo n.º 2
0
def populate_db():
    db_create()
    ucontr = UserController()
    ccontr = CategoryController()
    fcontr = FeedController()
    acontr = ArticleController()
    ccontr = CategoryController()
    user1, user2 = [ucontr.create(login=name, email="*****@*****.**" % name,
                                  password=name)
                    for name in ["user1", "user2"]]

    article_total = 0
    for user in (user1, user2):
        for i in range(3):
            cat_id = None
            if i:
                cat_id = ccontr.create(user_id=user.id,
                                       name="category%d" % i).id
            feed = fcontr.create(link="feed%d" % i, user_id=user.id,
                                    category_id=cat_id,
                                    title="%s feed%d" % (user.login, i))
            for j in range(3):
                entry = "%s %s article%d" % (user.login, feed.title, j)
                article_total += 1
                acontr.create(entry_id=entry,
                        link='http://test.te/%d' % article_total,
                        feed_id=feed.id, user_id=user.id, category_id=cat_id,
                        title=entry, content="content %d" % article_total)
Exemplo n.º 3
0
async def insert_articles(queue, nḅ_producers=1):
    """Consumer coroutines.
    """
    nb_producers_done = 0
    while True:
        item = await queue.get()
        if item is None:
            nb_producers_done += 1
            if nb_producers_done == nḅ_producers:
                print('All producers done.')
                print('Process finished.')
                break
            continue

        user, feed, articles = item


        if None is articles:
            logger.info('None')
            articles = []

        logger.info('Inserting articles for {}'.format(feed.link))

        art_contr = ArticleController(user.id)
        for article in articles:
            new_article = await construct_article(article, feed)

            try:
                existing_article_req = art_contr.read(
                                        user_id=user.id,
                                        feed_id=feed.id,
                                        entry_id=extract_id(article))
            except Exception as e:
                logger.exception("existing_article_req: " + str(e))
                continue
            exist = existing_article_req.count() != 0
            if exist:
                continue

            # insertion of the new article
            try:
                art_contr.create(**new_article)
                logger.info('New article added: {}'.format(new_article['link']))
            except Exception:
                logger.exception('Error when inserting article in database.')
                continue
Exemplo n.º 4
0
async def insert_database(user, feed):

    articles = await parse_feed(user, feed)
    if None is articles:
        return []

    logger.debug('inserting articles for {}'.format(feed.title))

    logger.info("Database insertion...")
    new_articles = []
    art_contr = ArticleController(user.id)
    for article in articles:
        existing_article_req = art_contr.read(feed_id=feed.id,
                                              **extract_id(article))
        exist = existing_article_req.count() != 0
        if exist:
            # if the article has been already retrieved, we only update
            # the content or the title
            logger.debug("Article %r (%r) already in the database.",
                         article['title'], article['link'])
            existing_article = existing_article_req.first()
            new_updated_date = None
            try:
                new_updated_date = dateutil.parser.parse(article['updated'])
            except Exception as e:
                print(e)  #new_updated_date = existing_article.date
            if None is existing_article.updated_date:
                existing_article.updated_date = new_updated_date.replace(
                    tzinfo=None)
            if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \
                                new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'):
                existing_article.updated_date = \
                                        new_updated_date.replace(tzinfo=None)
                if existing_article.title != article['title']:
                    existing_article.title = article['title']
                content = get_article_content(article)
                if existing_article.content != content:
                    existing_article.content = content
                    existing_article.readed = False
                art_contr.update({'entry_id': existing_article.entry_id},
                                 existing_article.dump())
            continue
        article = construct_article(article, feed)
        try:
            new_articles.append(art_contr.create(**article))
            logger.info("New article % (%r) added.", article['title'],
                        article['link'])
        except Exception:
            logger.exception("Error when inserting article in database:")
            continue
    return new_articles
Exemplo n.º 5
0
async def insert_database(user, feed):
    articles = await parse_feed(user, feed)
    if None is articles:
        return []

    logger.info('Inserting articles for {}'.format(feed.title))

    new_articles = []
    art_contr = ArticleController(user.id)
    for article in articles:
        new_article = await construct_article(article, feed)

        try:
            existing_article_req = art_contr.read(feed_id=feed.id,
                            entry_id=extract_id(article))
        except Exception as e:
            logger.exception("existing_article_req: " + str(e))
            continue
        exist = existing_article_req.count() != 0
        if exist:
            continue
            # if the article has been already retrieved, we only update
            # the content or the title
            logger.info('Article already in the database: {}'. \
                            format(article['link']))
            existing_article = existing_article_req.first()

            if new_article['date'].replace(tzinfo=None) != \
                                                        existing_article.date:
                existing_article.date = new_article['date']
                existing_article.updated_date = new_article['date']
                if existing_article.title != new_article['title']:
                    existing_article.title = new_article['title']
                content = get_article_content(article)
                if existing_article.content != content:
                    existing_article.content = content
                    existing_article.readed = False
                art_contr.update({'entry_id': existing_article.entry_id},
                                                        existing_article.dump())
                logger.info('Article updated: {}'.format(article['link']))
            continue

        # insertion of the new article
        try:
            new_articles.append(art_contr.create(**new_article))
            logger.info('New article added: {}'.format(new_article['link']))
        except Exception:
            logger.exception('Error when inserting article in database:')
            continue
    return new_articles
Exemplo n.º 6
0
async def insert_database(user, feed):

    articles = await parse_feed(user, feed)
    if None is articles:
        return []

    logger.debug('inserting articles for {}'.format(feed.title))

    logger.info("Database insertion...")
    new_articles = []
    art_contr = ArticleController(user.id)
    for article in articles:
        existing_article_req = art_contr.read(feed_id=feed.id,
                        **extract_id(article))
        exist = existing_article_req.count() != 0
        if exist:
            # if the article has been already retrieved, we only update
            # the content or the title
            logger.debug("Article %r (%r) already in the database.",
                         article['title'], article['link'])
            existing_article = existing_article_req.first()
            new_updated_date = None
            try:
                new_updated_date = dateutil.parser.parse(article['updated'])
            except Exception as e:
                print(e)#new_updated_date = existing_article.date
            if None is existing_article.updated_date:
                existing_article.updated_date = new_updated_date.replace(tzinfo=None)
            if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \
                                new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'):
                existing_article.updated_date = \
                                        new_updated_date.replace(tzinfo=None)
                if existing_article.title != article['title']:
                    existing_article.title = article['title']
                content = get_article_content(article)
                if existing_article.content != content:
                    existing_article.content = content
                    existing_article.readed = False
                art_contr.update({'entry_id': existing_article.entry_id},
                                                        existing_article.dump())
            continue
        article = construct_article(article, feed)
        try:
            new_articles.append(art_contr.create(**article))
            logger.info("New article % (%r) added.",
                        article['title'], article['link'])
        except Exception:
            logger.exception("Error when inserting article in database:")
            continue
    return new_articles
Exemplo n.º 7
0
async def insert_database(user, feed):
    articles = await parse_feed(user, feed)
    if None is articles:
        return []

    logger.info('Inserting articles for {}'.format(feed.title))

    new_articles = []
    art_contr = ArticleController(user.id)
    for article in articles:
        new_article = await construct_article(article, feed)

        try:
            existing_article_req = art_contr.read(feed_id=feed.id,
                            entry_id=extract_id(article))
        except Exception as e:
            logger.exception("existing_article_req: " + str(e))
            continue
        exist = existing_article_req.count() != 0
        if exist:
            # if the article has been already retrieved, we only update
            # the content or the title
            logger.info('Article already in the database: {}'. \
                            format(article['link']))
            existing_article = existing_article_req.first()

            if new_article['date'].replace(tzinfo=None) != \
                                                        existing_article.date:
                existing_article.date = new_article['date']
                existing_article.updated_date = new_article['date']
                if existing_article.title != new_article['title']:
                    existing_article.title = new_article['title']
                content = get_article_content(article)
                if existing_article.content != content:
                    existing_article.content = content
                    existing_article.readed = False
                art_contr.update({'entry_id': existing_article.entry_id},
                                                        existing_article.dump())
                logger.info('Article updated: {}'.format(article['link']))
            continue

        # insertion of the new article
        try:
            new_articles.append(art_contr.create(**new_article))
            logger.info('New article added: {}'.format(new_article['link']))
        except Exception:
            logger.exception('Error when inserting article in database:')
            continue
    return new_articles
Exemplo n.º 8
0
async def insert_database(user, feed):

    articles = await parse_feed(user, feed)
    if None is articles:
        return []

    logger.debug('inserting articles for {}'.format(feed.title))

    logger.info("Database insertion...")
    new_articles = []
    art_contr = ArticleController(user.id)
    for article in articles:
        existing_article_req = art_contr.read(feed_id=feed.id,
                        **extract_id(article))
        exist = existing_article_req.count() != 0
        if exist:
            existing_article = existing_article_req.first()
            is_updated = False
            logger.debug("Article %r (%r) already in the database.",
                         article['title'], article['link'])
            content = get_article_content(article)
            if existing_article.title != article['title']:
                existing_article.title = article['title']
                is_updated = True
            if existing_article.content != content:
                existing_article.content = content
                existing_article.readed = False
                is_updated = True
            if is_updated:
                art_contr.update({'entry_id': existing_article.entry_id},
                                 existing_article.dump())
            continue
        article = construct_article(article, feed.dump())
        try:
            new_articles.append(art_contr.create(**article))
            logger.info("New article % (%r) added.",
                        article['title'], article['link'])
        except Exception:
            logger.exception("Error when inserting article in database:")
            continue
    return new_articles
Exemplo n.º 9
0
async def insert_database(user, feed):

    articles = await parse_feed(user, feed)
    if None is articles:
        return []

    logger.debug('inserting articles for {}'.format(feed.title))

    logger.info("Database insertion...")
    new_articles = []
    art_contr = ArticleController(user.id)
    for article in articles:
        existing_article_req = art_contr.read(feed_id=feed.id,
                                              **extract_id(article))
        exist = existing_article_req.count() != 0
        if exist:
            existing_article = existing_article_req.first()
            is_updated = False
            logger.debug("Article %r (%r) already in the database.",
                         article['title'], article['link'])
            content = get_article_content(article)
            if existing_article.title != article['title']:
                existing_article.title = article['title']
                is_updated = True
            if existing_article.content != content:
                existing_article.content = content
                existing_article.readed = False
                is_updated = True
            if is_updated:
                art_contr.update({'entry_id': existing_article.entry_id},
                                 existing_article.dump())
            continue
        article = construct_article(article, feed.dump())
        try:
            new_articles.append(art_contr.create(**article))
            logger.info("New article % (%r) added.", article['title'],
                        article['link'])
        except Exception:
            logger.exception("Error when inserting article in database:")
            continue
    return new_articles