コード例 #1
0
 def testWalkerOnlyIngestsNewEntries(self):
     '''
     Test that the walker will stop when it gets to an entry that isn't new.
     '''
     # We build a persister which says there are three entries
     # that aren't in the repository.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True, True, True, False).one_by_one.times(4)
     persister.should_receive('process').with_args(object, object).times(3)
     parser = AtomWalker(
         'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
         persister)
     parser.ingest()
     # We build a persister which says there are two entries
     # that aren't in the repository.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True, True, False, False).one_by_one.times(4)
     persister.should_receive('process').with_args(object, object).times(2)
     parser = AtomWalker(
         'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
         persister)
     parser.ingest()
     # We build a persister which says there is one entry
     # that isn't in the repository.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True, False, False, False).one_by_one.times(2)
     persister.should_receive('process').with_args(object, object).times(1)
     parser = AtomWalker(
         'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
         persister)
     parser.ingest()
コード例 #2
0
 def testWalkerOnlyIngestsNewEntries(self):
     '''
     Test that the walker will stop when it gets to an entry that isn't new.
     '''
     # We build a persister which says there are three entries
     # that aren't in the repository.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True, True, True, False).one_by_one.times(4)
     persister.should_receive('process').with_args(object, object).times(3)
     parser = AtomWalker('http://localhost:%d/datasets.atom' %
                         (TestWebServer.getPort()),
                         persister)
     parser.ingest()
     # We build a persister which says there are two entries
     # that aren't in the repository.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True, True, False, False).one_by_one.times(4)
     persister.should_receive('process').with_args(object, object).times(2)
     parser = AtomWalker('http://localhost:%d/datasets.atom' %
                         (TestWebServer.getPort()),
                         persister)
     parser.ingest()
     # We build a persister which says there is one entry
     # that isn't in the repository.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True, False, False, False).one_by_one.times(2)
     persister.should_receive('process').with_args(object, object).times(1)
     parser = AtomWalker('http://localhost:%d/datasets.atom' %
                         (TestWebServer.getPort()),
                         persister)
     parser.ingest()
コード例 #3
0
 def testWalkerProcessesEntriesInCorrectOrder(self):
     '''
     Test that the walker processes the entries in the revese order that it
     finds them.
     '''
     checked_entries = []
     processed_entries = []
     # We build a persister which says all entries are new.
     persister = flexmock(AtomPersister())
     # Grab the checked entry and return true
     persister.should_receive('is_new').with_args(object, object)\
         .replace_with(lambda feed, entry: checked_entries.append(entry) or True)
     # Grab the processed entry
     persister.should_receive('process').with_args(object, object)\
         .replace_with(lambda feed, entry: processed_entries.append(entry))
     parser = AtomWalker('http://localhost:%d/datasets.atom' %
                         (TestWebServer.getPort()),
                         persister)
     parser.ingest()
     # We should have checked four entries, chronologically decendent
     eq_(len(checked_entries), 4)
     checked_backwards = reduce(self._check_chronological_asc_order,\
                                  reversed(checked_entries), None)
     ok_(checked_backwards, "Entries aren't parsed in reverse chronological order")
     # We should have processed four entries, chronologically ascendent
     eq_(len(processed_entries), 4)
     processed_forwards = reduce(self._check_chronological_asc_order,\
                                  processed_entries, None)
     ok_(processed_forwards, "Entries aren't parsed in chronological order")
コード例 #4
0
 def testWalkerProcessesEntriesInCorrectOrder(self):
     '''
     Test that the walker processes the entries in the revese order that it
     finds them.
     '''
     checked_entries = []
     processed_entries = []
     # We build a persister which says all entries are new.
     persister = flexmock(AtomPersister())
     # Grab the checked entry and return true
     persister.should_receive('is_new').with_args(object, object)\
         .replace_with(lambda feed, entry: checked_entries.append(entry) or True)
     # Grab the processed entry
     persister.should_receive('process').with_args(object, object)\
         .replace_with(lambda feed, entry: processed_entries.append(entry))
     parser = AtomWalker(
         'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
         persister)
     parser.ingest()
     # We should have checked four entries, chronologically decendent
     eq_(len(checked_entries), 4)
     checked_backwards = reduce(self._check_chronological_asc_order,\
                                  reversed(checked_entries), None)
     ok_(checked_backwards,
         "Entries aren't parsed in reverse chronological order")
     # We should have processed four entries, chronologically ascendent
     eq_(len(processed_entries), 4)
     processed_forwards = reduce(self._check_chronological_asc_order,\
                                  processed_entries, None)
     ok_(processed_forwards, "Entries aren't parsed in chronological order")
コード例 #5
0
    def setUpClass(cls):
        cls.priorcwd = os.getcwd()
        os.chdir(os.path.dirname(__file__) + '/atom_test')
        cls.server = TestWebServer()
        cls.server.start()

        Location.force_initialize()
        Location.load_location({
            'name': 'test-atom',
            'transfer_provider': 'http',
            'url': 'http://localhost:4272/files/',
            'type': 'external',
            'priority': 10
        })
        Location.load_location({
            'name': 'test-atom2',
            'transfer_provider': 'http',
            'url': 'http://mydatagrabber.cmm.uq.edu.au/files',
            'type': 'external',
            'priority': 10
        })

        files = path.realpath(
            path.join(path.dirname(__file__), 'atom_test', 'files'))
        Location.load_location({
            'name': 'test-atom3',
            'transfer_provider': 'local',
            'url': 'file://' + files,
            'type': 'external',
            'priority': 10
        })
コード例 #6
0
 def testWalkerFollowsAtomLinks(self):
     """
     Test that the walker follows links.
     """
     # We build a persister which says all entries are new.
     persister = flexmock(AtomPersister())
     persister.should_receive("is_new").with_args(object, object).and_return(True).times(4)
     persister.should_receive("process").with_args(object, object).times(4)
     walker = AtomWalker("http://localhost:%d/datasets.atom" % (TestWebServer.getPort()), persister)
     ok_(inspect.ismethod(walker.ingest), "Walker must have an ingest method")
     walker.ingest()
コード例 #7
0
 def testWalkerFollowsAtomLinks(self):
     '''
     Test that the walker follows links.
     '''
     # We build a persister which says all entries are new.
     persister = flexmock(AtomPersister())
     persister.should_receive('is_new').with_args(object, object)\
         .and_return(True).times(4)
     persister.should_receive('process').with_args(object, object)\
         .times(4)
     walker = AtomWalker(
         'http://localhost:%d/datasets.atom' % (TestWebServer.getPort()),
         persister)
     ok_(inspect.ismethod(walker.ingest),
         "Walker must have an ingest method")
     walker.ingest()