def build_file_error_from_response(self, response, **kwargs): item = FileError({ 'url': response.request.url, 'errors': { 'http_code': response.status }, }) if 'file_name' in response.request.meta: item['file_name'] = response.request.meta['file_name'] item.update(kwargs) return item
def build_file_error_from_response(self, response, **kwargs): """ Returns a FileError item to yield, based on the response to a request. """ item = FileError({ 'url': response.request.url, 'errors': {'http_code': response.status}, }) if 'file_name' in response.request.meta: item['file_name'] = response.request.meta['file_name'] item.update(kwargs) return item
def test_process_file_error(): pipeline = Validate() item = FileError({ 'file_name': 'test', 'url': 'http://test.com', 'errors': 'Error' }) assert pipeline.process_item(item, None) == item
def test_process_file_item_error_error(): pipeline = Validate() item = FileError({'file_name': 'test', 'url': 'http://test.com'}) with pytest.raises(ValidationError): pipeline.process_item(item, None) item['errors'] = 'Error' item['url'] = 'not an url' with pytest.raises(ValidationError): pipeline.process_item(item, None)
def test_process_item_file_error(): pipeline = Sample() spider = spider_with_crawler(sample=1) item = FileError({ 'file_name': 'test', 'url': 'http://test.com', 'errors': 'error', }) with pytest.raises(DropItem): pipeline.process_item(item, spider)
def build_file_error_from_response(self, response, **kwargs): file_error = { 'url': response.request.url, 'errors': { 'http_code': response.status }, } if 'kf_filename' in response.request.meta: file_error['file_name'] = response.request.meta['kf_filename'] file_error.update(kwargs) return FileError(file_error)
def test_file_error(): spider = spider_with_crawler() spider.latest = True pipeline = LatestReleaseDate() item = FileError({ 'file_name': 'test', 'url': 'http://test.com', 'errors': 'error' }) with pytest.raises(DropItem): pipeline.process_item(item, spider)
def test_item_scraped_file_error(sample, is_sample, ok, tmpdir, caplog): spider = spider_with_files_store(tmpdir, sample=sample) api_extension = KingfisherProcessAPI.from_crawler(spider.crawler) with patch('requests.post') as mocked: response = Mock() response.ok = ok response.status_code = 400 mocked.return_value = response data = FileError({ 'file_name': 'file.json', 'url': 'https://example.com/remote.json', 'errors': { 'http_code': 500 }, }) api_extension.item_scraped(data, spider) if not ok: message = 'Failed to post [https://example.com/remote.json]. File Errors API status code: 400' assert len(caplog.records) == 1 assert caplog.records[0].name == 'test' assert caplog.records[0].levelname == 'WARNING' assert caplog.records[0].message == message expected = { 'collection_source': 'test', 'collection_data_version': '2001-02-03 04:05:06', 'collection_sample': is_sample, 'file_name': 'file.json', 'url': 'https://example.com/remote.json', # Specific to this test case. 'errors': '{"http_code": 500}', } mocked.assert_called_once_with( 'http://httpbin.org/anything/api/v1/submit/file_errors/', headers={ 'Authorization': 'ApiKey xxx', }, proxies={ 'http': None, 'https': None, }, data=expected, )
def test_item_scraped_file_error(caplog): spider = spider_with_crawler() item_extension = KingfisherItemCount.from_crawler(spider.crawler) item = FileError({ 'url': 'https://example.com/remote.json', 'errors': { 'http_code': 404 }, }) item_extension.item_scraped(item, spider) assert item_extension.stats.get_value('fileerror_count') == 1 assert item_extension.stats.get_value('file_count', 0) == 0 assert item_extension.stats.get_value('fileitem_count', 0) == 0
def build_file(self, file_name=None, url=None, data=None, **kwargs): json_data = json.loads(data) # Some files contain invalid record packages, e.g.: # { # "status": 500, # "detail": "error" # } if 'status' in json_data and json_data['status'] != 200: json_data['http_code'] = json_data['status'] return FileError({ 'file_name': file_name, 'url': url, 'errors': json_data, }) else: return super().build_file(file_name=file_name, url=url, data=data, **kwargs)
def test_item_scraped_file_error(sample, is_sample, ok, tmpdir, caplog): with patch('treq.response._Response.code', new_callable=PropertyMock) as mocked: mocked.return_value = 200 if ok else 400 spider = spider_with_files_store(tmpdir, sample=sample) extension = KingfisherProcessAPI.from_crawler(spider.crawler) item = FileError({ 'file_name': 'file.json', 'url': 'https://example.com/remote.json', 'errors': { 'http_code': 500 }, }) response = yield extension.item_scraped(item, spider) data = yield response.json() form = { 'collection_source': 'test', 'collection_data_version': '2001-02-03 04:05:06', 'collection_sample': str(is_sample), 'file_name': 'file.json', 'url': 'https://example.com/remote.json', # Specific to FileError. 'errors': '{"http_code": 500}', } assert data['method'] == 'POST' assert data[ 'url'] == 'http://httpbin.org/anything/api/v1/submit/file_errors/' assert data['headers']['Authorization'] == 'ApiKey xxx' assert data['form'] == form assert data['args'] == {} assert data['data'] == '' assert data['files'] == {} if not ok: message = 'create_file_error failed (https://example.com/remote.json) with status code: 400' assert len(caplog.records) == 1 assert caplog.records[0].name == 'test' assert caplog.records[0].levelname == 'WARNING' assert caplog.records[0].message == message
AddPackageMiddleware, LineDelimitedMiddleware, ResizePackageMiddleware, RootPathMiddleware, ReadDataMiddleware, ]) @pytest.mark.parametrize('item', [ File({ 'file_name': 'test', 'data': 'data', 'data_type': 'release_package', 'url': 'http://test.com', }), FileError({ 'file_name': 'test', 'url': 'http://test.com', 'errors': '' }), ]) def test_yield_items(middleware_class, item): spider = spider_with_crawler() middleware = middleware_class() generator = middleware.process_spider_output(None, [item], spider) returned_item = next(generator) assert item == returned_item @pytest.mark.parametrize('data_type,data,root_path', [