def testWalkerOnlyIngestsNewEntries(self): ''' Test that the walker will stop when it gets to an entry that isn't new. ''' # We build a persister which says there are three entries # that aren't in the repository. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True, True, True, False).one_by_one.times(4) persister.should_receive('process').with_args(object, object).times(3) parser = AtomWalker( 'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest() # We build a persister which says there are two entries # that aren't in the repository. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True, True, False, False).one_by_one.times(4) persister.should_receive('process').with_args(object, object).times(2) parser = AtomWalker( 'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest() # We build a persister which says there is one entry # that isn't in the repository. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True, False, False, False).one_by_one.times(2) persister.should_receive('process').with_args(object, object).times(1) parser = AtomWalker( 'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest()
def testWalkerOnlyIngestsNewEntries(self): ''' Test that the walker will stop when it gets to an entry that isn't new. ''' # We build a persister which says there are three entries # that aren't in the repository. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True, True, True, False).one_by_one.times(4) persister.should_receive('process').with_args(object, object).times(3) parser = AtomWalker('http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest() # We build a persister which says there are two entries # that aren't in the repository. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True, True, False, False).one_by_one.times(4) persister.should_receive('process').with_args(object, object).times(2) parser = AtomWalker('http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest() # We build a persister which says there is one entry # that isn't in the repository. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True, False, False, False).one_by_one.times(2) persister.should_receive('process').with_args(object, object).times(1) parser = AtomWalker('http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest()
def testWalkerProcessesEntriesInCorrectOrder(self): ''' Test that the walker processes the entries in the revese order that it finds them. ''' checked_entries = [] processed_entries = [] # We build a persister which says all entries are new. persister = flexmock(AtomPersister()) # Grab the checked entry and return true persister.should_receive('is_new').with_args(object, object)\ .replace_with(lambda feed, entry: checked_entries.append(entry) or True) # Grab the processed entry persister.should_receive('process').with_args(object, object)\ .replace_with(lambda feed, entry: processed_entries.append(entry)) parser = AtomWalker('http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest() # We should have checked four entries, chronologically decendent eq_(len(checked_entries), 4) checked_backwards = reduce(self._check_chronological_asc_order,\ reversed(checked_entries), None) ok_(checked_backwards, "Entries aren't parsed in reverse chronological order") # We should have processed four entries, chronologically ascendent eq_(len(processed_entries), 4) processed_forwards = reduce(self._check_chronological_asc_order,\ processed_entries, None) ok_(processed_forwards, "Entries aren't parsed in chronological order")
def testWalkerProcessesEntriesInCorrectOrder(self): ''' Test that the walker processes the entries in the revese order that it finds them. ''' checked_entries = [] processed_entries = [] # We build a persister which says all entries are new. persister = flexmock(AtomPersister()) # Grab the checked entry and return true persister.should_receive('is_new').with_args(object, object)\ .replace_with(lambda feed, entry: checked_entries.append(entry) or True) # Grab the processed entry persister.should_receive('process').with_args(object, object)\ .replace_with(lambda feed, entry: processed_entries.append(entry)) parser = AtomWalker( 'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) parser.ingest() # We should have checked four entries, chronologically decendent eq_(len(checked_entries), 4) checked_backwards = reduce(self._check_chronological_asc_order,\ reversed(checked_entries), None) ok_(checked_backwards, "Entries aren't parsed in reverse chronological order") # We should have processed four entries, chronologically ascendent eq_(len(processed_entries), 4) processed_forwards = reduce(self._check_chronological_asc_order,\ processed_entries, None) ok_(processed_forwards, "Entries aren't parsed in chronological order")
def setUpClass(cls): cls.priorcwd = os.getcwd() os.chdir(os.path.dirname(__file__) + '/atom_test') cls.server = TestWebServer() cls.server.start() Location.force_initialize() Location.load_location({ 'name': 'test-atom', 'transfer_provider': 'http', 'url': 'http://localhost:4272/files/', 'type': 'external', 'priority': 10 }) Location.load_location({ 'name': 'test-atom2', 'transfer_provider': 'http', 'url': 'http://mydatagrabber.cmm.uq.edu.au/files', 'type': 'external', 'priority': 10 }) files = path.realpath( path.join(path.dirname(__file__), 'atom_test', 'files')) Location.load_location({ 'name': 'test-atom3', 'transfer_provider': 'local', 'url': 'file://' + files, 'type': 'external', 'priority': 10 })
def testWalkerFollowsAtomLinks(self): """ Test that the walker follows links. """ # We build a persister which says all entries are new. persister = flexmock(AtomPersister()) persister.should_receive("is_new").with_args(object, object).and_return(True).times(4) persister.should_receive("process").with_args(object, object).times(4) walker = AtomWalker("http://localhost:%d/datasets.atom" % (TestWebServer.getPort()), persister) ok_(inspect.ismethod(walker.ingest), "Walker must have an ingest method") walker.ingest()
def testWalkerFollowsAtomLinks(self): ''' Test that the walker follows links. ''' # We build a persister which says all entries are new. persister = flexmock(AtomPersister()) persister.should_receive('is_new').with_args(object, object)\ .and_return(True).times(4) persister.should_receive('process').with_args(object, object)\ .times(4) walker = AtomWalker( 'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()), persister) ok_(inspect.ismethod(walker.ingest), "Walker must have an ingest method") walker.ingest()