Пример #1
0
    def test_bulkdata(self):
        j = self.project.push_job(self.spidername, state='running')
        for i in xrange(20):
            j.logs.info("log line %d" % i)
            j.items.write(dict(field1="item%d" % i))
            j.requests.add("http://test.com/%d" % i,
                200, 'GET', 10, None, 10, 120)
        for resourcename in ('logs', 'items', 'requests'):
            resource = getattr(j, resourcename)
            resource.flush()

            # downloading resource, with simulated failures
            with failing_downloader(resource):
                downloaded = list(resource.iter_values())
                self.assertEqual(len(downloaded), 20)
Пример #2
0
    def test_bulkdata(self):
        j = self.project.push_job(self.spidername, state='running')
        for i in xrange(20):
            j.logs.info("log line %d" % i)
            j.items.write(dict(field1="item%d" % i))
            j.requests.add("http://test.com/%d" % i, 200, 'GET', 10, None, 10,
                           120)
        for resourcename in ('logs', 'items', 'requests'):
            resource = getattr(j, resourcename)
            resource.flush()

            # downloading resource, with simulated failures
            with failing_downloader(resource):
                downloaded = list(resource.iter_values())
                self.assertEqual(len(downloaded), 20)
Пример #3
0
    def test_data_download(self):
        col = self.project.collections.new_store(self.test_collection_name)
        items = []
        with closing(col.create_writer()) as writer:
            for i in xrange(20):
                test_item = _mkitem()
                test_item['_key'] = "test_data_download%d" % i
                test_item['counter'] = i
                writer.write(test_item)
                items.append(test_item)

        # check parameters are passed correctly
        downloaded = list(col.iter_values(prefix='test_data_download1'))
        self.assertEqual(len(downloaded), 11)

        # simulate network timeouts and download data
        with failing_downloader(self.project.collections):
            downloaded = list(col.iter_values(start='test_data_download1'))
            self.assertEqual(len(downloaded), 19)