def setUp(self): self.mocked_proc = mock.MagicMock() self.mocked_proc.communicate.return_value = ([], None) self.mocked_proc.wait.return_value = 0 self.mocked_popen = mock.Mock() self.mocked_popen.return_value = self.mocked_proc self.patch = mock.patch("subprocess.Popen", self.mocked_popen) crawler_mock = mock.Mock() crawler_mock.settings = Settings( { "DOTSCRAPY_ENABLED": True, "ADDONS_S3_BUCKET": "test-bucket", "ADDONS_AWS_ACCESS_KEY_ID": "access-key", "ADDONS_AWS_SECRET_ACCESS_KEY": "secret-key", "ADDONS_AWS_USERNAME": "******", } ) os.environ.update( { "SCRAPY_JOB": "123/45/67", "SCRAPY_PROJECT_ID": "123", "SCRAPY_SPIDER": "testspider", "HOME": "/home/user", "DOTSCRAPY_DIR": "/tmp/.scrapy", } ) self.patch.start() self.instance = DotScrapyPersistence.from_crawler(crawler_mock)
def test_from_crawler(self): crawler_mock = mock.Mock() crawler_mock.settings = Settings() self.assertRaises(NotConfigured, DotScrapyPersistence.from_crawler, crawler_mock) # add needed settings for from_crawler() crawler_mock.settings.set("DOTSCRAPY_ENABLED", True) crawler_mock.settings.set("ADDONS_S3_BUCKET", "s3-test-bucket") instance = DotScrapyPersistence.from_crawler(crawler_mock) assert isinstance(instance, DotScrapyPersistence)
def test_from_crawler(self): crawler_mock = mock.Mock() crawler_mock.settings = Settings() self.assertRaises(NotConfigured, DotScrapyPersistence.from_crawler, crawler_mock) # add needed settings for from_crawler() crawler_mock.settings.set('DOTSCRAPY_ENABLED', True) crawler_mock.settings.set('ADDONS_S3_BUCKET', 's3-test-bucket') instance = DotScrapyPersistence.from_crawler(crawler_mock) assert isinstance(instance, DotScrapyPersistence)
def setUp(self): self.mocked_proc = mock.MagicMock() self.mocked_proc.communicate.return_value = ([], None) self.mocked_proc.wait.return_value = 0 self.mocked_popen = mock.Mock() self.mocked_popen.return_value = self.mocked_proc self.patch = mock.patch('subprocess.Popen', self.mocked_popen) crawler_mock = mock.Mock() crawler_mock.settings = Settings({ 'DOTSCRAPY_ENABLED': True, 'ADDONS_S3_BUCKET': 'test-bucket', 'ADDONS_AWS_ACCESS_KEY_ID': 'access-key', 'ADDONS_AWS_SECRET_ACCESS_KEY': 'secret-key', 'ADDONS_AWS_USERNAME': '******', }) os.environ.update({ 'SCRAPY_JOB': '123/45/67', 'SCRAPY_PROJECT_ID': '123', 'SCRAPY_SPIDER': 'testspider', 'HOME': '/home/user', 'DOTSCRAPY_DIR': '/tmp/.scrapy', }) self.patch.start() self.instance = DotScrapyPersistence.from_crawler(crawler_mock)