def testWalkerOnlyIngestsNewEntries(self):
'''
Test that the walker will stop when it gets to an entry that isn't new.
'''
# We build a persister which says there are three entries
# that aren't in the repository.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True, True, True, False).one_by_one.times(4)
persister.should_receive('process').with_args(object, object).times(3)
parser = AtomWalker('http://localhost:%d/datasets.atom' %
(TestWebServer.getPort()),
persister)
parser.ingest()
# We build a persister which says there are two entries
# that aren't in the repository.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True, True, False, False).one_by_one.times(4)
persister.should_receive('process').with_args(object, object).times(2)
parser = AtomWalker('http://localhost:%d/datasets.atom' %
(TestWebServer.getPort()),
persister)
parser.ingest()
# We build a persister which says there is one entry
# that isn't in the repository.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True, False, False, False).one_by_one.times(2)
persister.should_receive('process').with_args(object, object).times(1)
parser = AtomWalker('http://localhost:%d/datasets.atom' %
(TestWebServer.getPort()),
persister)
parser.ingest()
def testWalkerOnlyIngestsNewEntries(self):
'''
Test that the walker will stop when it gets to an entry that isn't new.
'''
# We build a persister which says there are three entries
# that aren't in the repository.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True, True, True, False).one_by_one.times(4)
persister.should_receive('process').with_args(object, object).times(3)
parser = AtomWalker(
'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
persister)
parser.ingest()
# We build a persister which says there are two entries
# that aren't in the repository.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True, True, False, False).one_by_one.times(4)
persister.should_receive('process').with_args(object, object).times(2)
parser = AtomWalker(
'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
persister)
parser.ingest()
# We build a persister which says there is one entry
# that isn't in the repository.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True, False, False, False).one_by_one.times(2)
persister.should_receive('process').with_args(object, object).times(1)
parser = AtomWalker(
'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
persister)
parser.ingest()
def testWalkerProcessesEntriesInCorrectOrder(self):
'''
Test that the walker processes the entries in the revese order that it
finds them.
'''
checked_entries = []
processed_entries = []
# We build a persister which says all entries are new.
persister = flexmock(AtomPersister())
# Grab the checked entry and return true
persister.should_receive('is_new').with_args(object, object)\
.replace_with(lambda feed, entry: checked_entries.append(entry) or True)
# Grab the processed entry
persister.should_receive('process').with_args(object, object)\
.replace_with(lambda feed, entry: processed_entries.append(entry))
parser = AtomWalker('http://localhost:%d/datasets.atom' %
(TestWebServer.getPort()),
persister)
parser.ingest()
# We should have checked four entries, chronologically decendent
eq_(len(checked_entries), 4)
checked_backwards = reduce(self._check_chronological_asc_order,\
reversed(checked_entries), None)
ok_(checked_backwards, "Entries aren't parsed in reverse chronological order")
# We should have processed four entries, chronologically ascendent
eq_(len(processed_entries), 4)
processed_forwards = reduce(self._check_chronological_asc_order,\
processed_entries, None)
ok_(processed_forwards, "Entries aren't parsed in chronological order")
def testWalkerProcessesEntriesInCorrectOrder(self):
'''
Test that the walker processes the entries in the revese order that it
finds them.
'''
checked_entries = []
processed_entries = []
# We build a persister which says all entries are new.
persister = flexmock(AtomPersister())
# Grab the checked entry and return true
persister.should_receive('is_new').with_args(object, object)\
.replace_with(lambda feed, entry: checked_entries.append(entry) or True)
# Grab the processed entry
persister.should_receive('process').with_args(object, object)\
.replace_with(lambda feed, entry: processed_entries.append(entry))
parser = AtomWalker(
'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
persister)
parser.ingest()
# We should have checked four entries, chronologically decendent
eq_(len(checked_entries), 4)
checked_backwards = reduce(self._check_chronological_asc_order,\
reversed(checked_entries), None)
ok_(checked_backwards,
"Entries aren't parsed in reverse chronological order")
# We should have processed four entries, chronologically ascendent
eq_(len(processed_entries), 4)
processed_forwards = reduce(self._check_chronological_asc_order,\
processed_entries, None)
ok_(processed_forwards, "Entries aren't parsed in chronological order")
def testWalkerFollowsAtomLinks(self):
"""
Test that the walker follows links.
"""
# We build a persister which says all entries are new.
persister = flexmock(AtomPersister())
persister.should_receive("is_new").with_args(object, object).and_return(True).times(4)
persister.should_receive("process").with_args(object, object).times(4)
walker = AtomWalker("http://localhost:%d/datasets.atom" % (TestWebServer.getPort()), persister)
ok_(inspect.ismethod(walker.ingest), "Walker must have an ingest method")
walker.ingest()
def testWalkerFollowsAtomLinks(self):
'''
Test that the walker follows links.
'''
# We build a persister which says all entries are new.
persister = flexmock(AtomPersister())
persister.should_receive('is_new').with_args(object, object)\
.and_return(True).times(4)
persister.should_receive('process').with_args(object, object)\
.times(4)
walker = AtomWalker(
'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
persister)
ok_(inspect.ismethod(walker.ingest),
"Walker must have an ingest method")
walker.ingest()