Exemple #1
0
def check_iter_locked(db_path, pre_stuff, iter_stuff):
    """Actual implementation of test_errors_locked, so it can be reused."""

    # WAL provides more concurrency; some things won't to block with it enabled.
    storage = Storage(db_path, wal_enabled=False)

    feed = FeedData('one')
    entry = EntryData('one', 'entry', datetime(2010, 1, 1), title='entry')
    storage.add_feed(feed.url, datetime(2010, 1, 2))
    storage.add_or_update_entry(
        EntryUpdateIntent(entry, entry.updated, datetime(2010, 1, 1), 0, 0))
    storage.add_feed('two', datetime(2010, 1, 1))
    storage.add_or_update_entry(
        EntryUpdateIntent(entry._replace(feed_url='two'), entry.updated,
                          datetime(2010, 1, 1), 0, 0))
    storage.set_feed_metadata('two', '1', 1)
    storage.set_feed_metadata('two', '2', 2)
    storage.add_feed_tag('two', '1')
    storage.add_feed_tag('two', '2')

    if pre_stuff:
        pre_stuff(storage)

    rv = iter_stuff(storage)
    next(rv)

    # shouldn't raise an exception
    storage = Storage(db_path, timeout=0, wal_enabled=False)
    storage.mark_as_read_unread(feed.url, entry.id, 1)
    storage = Storage(db_path, timeout=0)
    storage.mark_as_read_unread(feed.url, entry.id, 0)
Exemple #2
0
def storage_with_two_entries(storage):
    storage.add_feed('feed', datetime(2010, 1, 1))
    storage.add_or_update_entry(
        EntryUpdateIntent(
            EntryData('feed', 'one', datetime(2010, 1, 1)),
            datetime(2010, 1, 2),
            datetime(2010, 1, 2),
            0,
        ))
    storage.add_or_update_entry(
        EntryUpdateIntent(
            EntryData('feed', 'two', datetime(2010, 1, 1)),
            datetime(2010, 1, 2),
            datetime(2010, 1, 2),
            1,
        ))
    return storage
Exemple #3
0
def test_object_id():
    assert Feed('url').object_id == 'url'
    assert Entry('entry', 'updated',
                 feed=Feed('url')).object_id == ('url', 'entry')
    assert EntrySearchResult('url', 'entry').object_id == ('url', 'entry')
    assert FeedData('url').object_id == 'url'
    assert EntryData('url', 'entry', 'updated').object_id == ('url', 'entry')
    assert FeedError('url').object_id == 'url'
    assert EntryError('url', 'entry').object_id == ('url', 'entry')
Exemple #4
0
def _make_entry(feed_number, number, updated, **kwargs):
    return EntryData(
        f'{feed_number}',
        # evals to tuple
        f'{feed_number}, {number}',
        updated,
        kwargs.pop('title', f'Entry #{number}'),
        kwargs.pop('link', f'http://www.example.com/entries/{number}'),
        **kwargs,
    )
Exemple #5
0
def test_important_entry_remains_important_after_update(storage):
    storage.mark_as_important_unimportant('feed', 'one', True)

    storage.add_or_update_entry(
        EntryUpdateIntent(
            EntryData('feed', 'one', datetime(2010, 1, 1)),
            datetime(2010, 1, 2),
            datetime(2010, 1, 2),
            0,
        ))

    assert {
        e.id
        for e in storage.get_entries(datetime(2010, 1, 1),
                                     EntryFilterOptions(important=True))
    } == {'one'}
Exemple #6
0
def test_entry_remains_read_after_update(storage_with_two_entries):
    storage = storage_with_two_entries
    storage.mark_as_read_unread('feed', 'one', True)

    storage.add_or_update_entry(
        EntryUpdateIntent(
            EntryData('feed', 'one', datetime(2010, 1, 1)),
            datetime(2010, 1, 2),
            datetime(2010, 1, 2),
            0,
        ))

    assert {
        e.id
        for e in storage.get_entries(datetime(2010, 1, 1),
                                     EntryFilterOptions(read=True))
    } == {'one'}
Exemple #7
0
def make_entries(feed_url, url, soup):
    for title, fragment, content in extract_text(soup):
        try:
            updated = datetime.strptime(title.split()[0], '%Y-%m-%d')
        except (ValueError, IndexError):
            continue

        link = urlunparse(urlparse(url)._replace(fragment=fragment))

        yield EntryData(
            feed_url=feed_url,
            id=title,
            updated=updated,
            title=title,
            link=link,
            summary=content,
        )
Exemple #8
0
def test_get_entries_for_update(storage_cls):
    storage = storage_cls(':memory:')
    storage.add_feed('feed', datetime(2010, 1, 1))
    storage.add_or_update_entry(
        EntryUpdateIntent(
            EntryData('feed', 'one', datetime(2010, 1, 1)),
            datetime(2010, 1, 2),
            datetime(2010, 1, 1),
            0,
        ))

    assert list(
        storage.get_entries_for_update([
            ('feed', 'one'), ('feed', 'two')
        ])) == [
            EntryForUpdate(datetime(2010, 1, 1)),
            None,
        ]
Exemple #9
0
def check_errors_locked(db_path, pre_stuff, do_stuff, exc_type):
    """Actual implementation of test_errors_locked, so it can be reused."""

    # WAL provides more concurrency; some things won't to block with it enabled.
    storage = Storage(db_path, wal_enabled=False)
    storage.db.execute("PRAGMA busy_timeout = 0;")

    feed = FeedData('one')
    entry = EntryData('one', 'entry', datetime(2010, 1, 2))
    storage.add_feed(feed.url, datetime(2010, 1, 1))
    storage.add_or_update_entry(
        EntryUpdateIntent(entry, entry.updated, datetime(2010, 1, 1), 0, 0))

    in_transaction = threading.Event()
    can_return_from_transaction = threading.Event()

    def target():
        storage = Storage(db_path, wal_enabled=False)
        storage.db.isolation_level = None
        storage.db.execute("BEGIN EXCLUSIVE;")
        in_transaction.set()
        can_return_from_transaction.wait()
        storage.db.execute("ROLLBACK;")

    if pre_stuff:
        pre_stuff(storage, feed, entry)

    thread = threading.Thread(target=target)
    thread.start()

    in_transaction.wait()

    try:
        with pytest.raises(exc_type) as excinfo:
            do_stuff(storage, feed, entry)
        assert 'locked' in str(excinfo.value.__cause__)
    finally:
        can_return_from_transaction.set()
        thread.join()
Exemple #10
0
import datetime

from reader import Content
from reader import Enclosure
from reader._types import EntryData
from reader._types import FeedData

feed = FeedData(url='{}invalid.json'.format(url_base), )

entries = [
    EntryData(
        feed_url=feed.url,
        id='2',
        updated=None,
        enclosures=(
            Enclosure(href='control'),
            Enclosure(href='float size', length=100),
            Enclosure(href='non-number size'),
        ),
    ),
    EntryData(
        feed_url=feed.url,
        id='3.1415',
        title='float id',
        updated=None,
    ),
    EntryData(
        feed_url=feed.url,
        id='author name',
        updated=None,
    ),
Exemple #11
0
    title='RSS Title',
    link='http://www.example.com/main.html',
    author='Example editor ([email protected])',
)

entries = [
    EntryData(
        feed_url=feed.url,
        id='7bd204c6-1655-4c27-aeee-53f933c5395f',
        updated=datetime.datetime(2009, 9, 6, 16, 20),
        title='Example entry',
        link='http://www.example.com/blog/post/1',
        author='Example editor',
        published=None,
        summary='Here is some text containing an interesting description.',
        content=(
            # the text/plain type comes from feedparser
            Content(value='Example content', type='text/plain'),
        ),
        enclosures=(
            Enclosure(href='http://example.com/enclosure'),
            Enclosure(href='http://example.com/enclosure-with-type', type='image/jpeg'),
            Enclosure(href='http://example.com/enclosure-with-length', length=100000),
            Enclosure(href='http://example.com/enclosure-with-bad-length'),
        ),
    ),
    EntryData(
        feed_url=feed.url,
        id='00000000-1655-4c27-aeee-00000000',
        updated=datetime.datetime(2009, 9, 6, 0, 0, 0),
        title='Example entry, again',
    ),
Exemple #12
0
import datetime

from reader import Content
from reader import Enclosure
from reader._types import EntryData
from reader._types import FeedData

feed = FeedData(url='{}empty.atom'.format(url_base))

entries = [
    EntryData(
        feed_url=feed.url,
        id='urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a',
        updated=None,
        # added by feedparser
        link='urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a',
    )
]
Exemple #13
0
import datetime

from reader import Content
from reader import Enclosure
from reader._types import EntryData
from reader._types import FeedData

feed = FeedData(url='{}relative.rss'.format(url_base),
                link='{}file.html'.format(rel_base))

entries = [
    EntryData(
        feed_url=feed.url,
        id='7bd204c6-1655-4c27-aeee-53f933c5395f',
        updated=None,
        link='{}blog/post/1'.format(rel_base),
        summary='one <a href="{}target">two</a> three'.format(rel_base),
        content=(
            Content(value='<script>evil</script> content',
                    type='text/plain',
                    language=None),
            Content(value='content', type='text/html', language=None),
        ),
        enclosures=(
            # for RSS feedparser doesn't make relative links absolute
            # (it does for Atom)
            Enclosure(href='enclosure?q=a#fragment'), ),
    )
]
Exemple #14
0
import datetime

from reader import Content
from reader import Enclosure
from reader._types import EntryData
from reader._types import FeedData

feed = FeedData(url='{}relative.atom'.format(url_base),
                link='{}file.html'.format(rel_base))

entries = [
    EntryData(
        feed_url=feed.url,
        id='urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a',
        updated=None,
        link='{}entries/entry.html'.format(rel_base),
        summary='one <a href="{}target">two</a> three'.format(rel_base),
        content=(
            Content(value='<script>evil</script> content',
                    type='text/plain',
                    language=None),
            Content(value='content', type='text/html', language=None),
        ),
        enclosures=(
            # the text/html type comes from feedparser
            Enclosure(href='{}enclosure?q=a#fragment'.format(rel_base),
                      type='text/html'), ),
    )
]
Exemple #15
0
import datetime

from reader import Content
from reader import Enclosure
from reader._types import EntryData
from reader._types import FeedData

feed = FeedData(url='{}empty.rss'.format(url_base))

entries = [
    EntryData(feed_url=feed.url,
              id='7bd204c6-1655-4c27-aeee-53f933c5395f',
              updated=None)
]
Exemple #16
0
import datetime

from reader import Content
from reader import Enclosure
from reader._types import EntryData
from reader._types import FeedData


feed = FeedData(
    url='{}empty.json'.format(url_base),
)

entries = [
    EntryData(
        feed_url=feed.url,
        id='1',
        updated=None,
        content=(
            Content(
                value='content',
                type='text/plain',
            ),
        ),
    ),
]
Exemple #17
0
 EntryData(
     feed_url=feed.url,
     id='urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a',
     updated=datetime.datetime(2003, 12, 13, 18, 30, 2),
     title='Atom-Powered Robots Run Amok',
     link='http://example.org/2003/12/13/atom03',
     author='John Doe',
     published=datetime.datetime(2003, 12, 13, 17, 17, 51),
     summary='Some text.',
     content=(
         # the text/plain type comes from feedparser
         Content(value='content', type='text/plain'),
         Content(value='content with type', type='text/whatever'),
         Content(value='content with lang',
                 type='text/plain',
                 language='en'),
     ),
     enclosures=(
         # the text/html type comes from feedparser
         Enclosure(href='http://example.org/enclosure', type='text/html'),
         Enclosure(href='http://example.org/enclosure-with-type',
                   type='text/whatever'),
         Enclosure(
             href='http://example.org/enclosure-with-length',
             type='text/html',
             length=1000,
         ),
         Enclosure(href='http://example.org/enclosure-with-bad-length',
                   type='text/html'),
     ),
 ),