Пример #1
0
 def test_files_result_field(self):
     another_pipeline = FilesPipeline.from_settings(
         Settings({
             'FILES_STORE': self.tempdir,
             'FILES_RESULT_FIELD': 'funny_field'
         }))
     self.assertEqual(self.pipeline.files_result_field,
                      self.default_settings.get('FILES_RESULT_FIELD'))
     self.assertEqual(another_pipeline.files_result_field, 'funny_field')
Пример #2
0
 def test_expires(self):
     another_pipeline = FilesPipeline.from_settings(
         Settings({
             'FILES_STORE': self.tempdir,
             'FILES_EXPIRES': 42
         }))
     self.assertEqual(self.pipeline.expires,
                      self.default_settings.getint('FILES_EXPIRES'))
     self.assertEqual(another_pipeline.expires, 42)
Пример #3
0
 def test_item_fields_default(self):
     url = 'http://www.example.com/files/1.txt'
     item = self.item_class(name='item1', file_urls=[url])
     pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
     requests = list(pipeline.get_media_requests(item, None))
     self.assertEqual(requests[0].url, url)
     results = [(True, {'url': url})]
     item = pipeline.item_completed(results, item, None)
     files = ItemAdapter(item).get("files")
     self.assertEqual(files, [results[0][1]])
     self.assertIsInstance(item, self.item_class)
 def test_different_settings_for_different_instances(self):
     """
     If there are different instances with different settings they should keep
     different settings.
     """
     custom_settings = self._generate_fake_settings()
     another_pipeline = FilesPipeline.from_settings(Settings(custom_settings))
     one_pipeline = FilesPipeline(self.tempdir)
     for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
         default_value = self.default_cls_settings[pipe_attr]
         self.assertEqual(getattr(one_pipeline, pipe_attr), default_value)
         custom_value = custom_settings[settings_attr]
         self.assertNotEqual(default_value, custom_value)
         self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value)
Пример #5
0
 def test_different_settings_for_different_instances(self):
     """
     If there are different instances with different settings they should keep
     different settings.
     """
     custom_settings = self._generate_fake_settings()
     another_pipeline = FilesPipeline.from_settings(Settings(custom_settings))
     one_pipeline = FilesPipeline(self.tempdir)
     for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
         default_value = self.default_cls_settings[pipe_attr]
         self.assertEqual(getattr(one_pipeline, pipe_attr), default_value)
         custom_value = custom_settings[settings_attr]
         self.assertNotEqual(default_value, custom_value)
         self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value)
    def test_item_fields_default(self):
        class TestItem(Item):
            name = Field()
            file_urls = Field()
            files = Field()

        for cls in TestItem, dict:
            url = 'http://www.example.com/files/1.txt'
            item = cls({'name': 'item1', 'file_urls': [url]})
            pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
            requests = list(pipeline.get_media_requests(item, None))
            self.assertEqual(requests[0].url, url)
            results = [(True, {'url': url})]
            pipeline.item_completed(results, item, None)
            self.assertEqual(item['files'], [results[0][1]])
Пример #7
0
    def test_item_fields_default(self):
        class TestItem(Item):
            name = Field()
            file_urls = Field()
            files = Field()

        for cls in TestItem, dict:
            url = 'http://www.example.com/files/1.txt'
            item = cls({'name': 'item1', 'file_urls': [url]})
            pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
            requests = list(pipeline.get_media_requests(item, None))
            self.assertEqual(requests[0].url, url)
            results = [(True, {'url': url})]
            pipeline.item_completed(results, item, None)
            self.assertEqual(item['files'], [results[0][1]])
Пример #8
0
    def test_item_fields_default(self):
        class TestItem(Item):
            name = Field()
            file_urls = Field()
            files = Field()

        for cls in TestItem, dict:
            url = "http://www.example.com/files/1.txt"
            item = cls({"name": "item1", "file_urls": [url]})
            pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": "s3://example/files/"}))
            requests = list(pipeline.get_media_requests(item, None))
            self.assertEqual(requests[0].url, url)
            results = [(True, {"url": url})]
            pipeline.item_completed(results, item, None)
            self.assertEqual(item["files"], [results[0][1]])
 def setUp(self):
     self.tempdir = mkdtemp()
     self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
     self.pipeline.download_func = _mocked_download_func
     self.pipeline.open_spider(None)
Пример #10
0
 def setUp(self):
     self.tempdir = mkdtemp()
     self.pipeline = FilesPipeline.from_settings(
         Settings({'FILES_STORE': self.tempdir}))
     self.pipeline.download_func = _mocked_download_func
     self.pipeline.open_spider(None)
Пример #11
0
 def test_files_result_field(self):
     another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
                                                             'FILES_RESULT_FIELD': 'funny_field'}))
     self.assertEqual(self.pipeline.files_result_field, self.default_settings.get('FILES_RESULT_FIELD'))
     self.assertEqual(another_pipeline.files_result_field, 'funny_field')
Пример #12
0
 def test_expires(self):
     another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir,
                                                             'FILES_EXPIRES': 42}))
     self.assertEqual(self.pipeline.expires, self.default_settings.getint('FILES_EXPIRES'))
     self.assertEqual(another_pipeline.expires, 42)