def test_different_settings_for_different_instances(self): """ If there are different instances with different settings they should keep different settings. """ custom_settings = self._generate_fake_settings() another_pipeline = FilesPipeline.from_settings(Settings(custom_settings)) one_pipeline = FilesPipeline(self.tempdir) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] self.assertEqual(getattr(one_pipeline, pipe_attr), default_value) custom_value = custom_settings[settings_attr] self.assertNotEqual(default_value, custom_value) self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value)
def setUp(self): self.tempdir = mkdtemp() settings_dict = {'FILES_STORE': self.tempdir} crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None)
def test_files_result_field(self): another_pipeline = FilesPipeline.from_settings( Settings({ 'FILES_STORE': self.tempdir, 'FILES_RESULT_FIELD': 'funny_field' })) self.assertEqual(self.pipeline.files_result_field, self.default_settings.get('FILES_RESULT_FIELD')) self.assertEqual(another_pipeline.files_result_field, 'funny_field')
def test_expires(self): another_pipeline = FilesPipeline.from_settings( Settings({ 'FILES_STORE': self.tempdir, 'FILES_EXPIRES': 42 })) self.assertEqual(self.pipeline.expires, self.default_settings.getint('FILES_EXPIRES')) self.assertEqual(another_pipeline.expires, 42)
def test_item_fields_default(self): url = 'http://www.example.com/files/1.txt' item = self.item_class(name='item1', file_urls=[url]) pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {'url': url})] item = pipeline.item_completed(results, item, None) files = ItemAdapter(item).get("files") self.assertEqual(files, [results[0][1]]) self.assertIsInstance(item, self.item_class)
def test_item_fields_default(self): class TestItem(Item): name = Field() file_urls = Field() files = Field() for cls in TestItem, dict: url = 'http://www.example.com/files/1.txt' item = cls({'name': 'item1', 'file_urls': [url]}) pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {'url': url})] pipeline.item_completed(results, item, None) self.assertEqual(item['files'], [results[0][1]])
def test_item_fields_default(self): class TestItem(Item): name = Field() file_urls = Field() files = Field() for cls in TestItem, dict: url = "http://www.example.com/files/1.txt" item = cls({"name": "item1", "file_urls": [url]}) pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": "s3://example/files/"})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {"url": url})] pipeline.item_completed(results, item, None) self.assertEqual(item["files"], [results[0][1]])
def setUp(self): self.tempdir = mkdtemp() self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None)
def setUp(self): self.tempdir = mkdtemp() self.pipeline = FilesPipeline.from_settings( Settings({'FILES_STORE': self.tempdir})) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None)
def __init__(self, store_uri, download_func=None, settings=None): FilesPipeline.__init__(self, store_uri, self.download, settings)
def test_files_result_field(self): another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir, 'FILES_RESULT_FIELD': 'funny_field'})) self.assertEqual(self.pipeline.files_result_field, self.default_settings.get('FILES_RESULT_FIELD')) self.assertEqual(another_pipeline.files_result_field, 'funny_field')
def test_expires(self): another_pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir, 'FILES_EXPIRES': 42})) self.assertEqual(self.pipeline.expires, self.default_settings.getint('FILES_EXPIRES')) self.assertEqual(another_pipeline.expires, 42)
def setUp(self): self.tempdir = mkdtemp() self.pipeline = FilesPipeline(self.tempdir) self.default_settings = Settings()