def fetch(self, request): '''Fulfill a request. Args: request (:class:`.ftp.request.Request`): Request. Returns: .ftp.request.Response: A Response populated with the initial data connection reply. Once the response is received, call :meth:`read_content`. Coroutine. ''' response = Response() yield From(self._prepare_fetch(request, response)) response.file_transfer_size = yield From(self._fetch_size(request)) if request.restart_value: try: yield From(self._commander.restart(request.restart_value)) response.restart_value = request.restart_value except FTPServerError: _logger.debug('Could not restart file.', exc_info=1) yield From(self._open_data_stream()) command = Command('RETR', request.file_path) yield From(self._begin_stream(command)) raise Return(response)
def test_to_dict(self): request = Request('ftp://foofle.com') request_dict = request.to_dict() self.assertEqual('ftp://foofle.com', request_dict['url']) self.assertEqual('ftp', request_dict['protocol']) response = Response() response.request = request response.reply = Reply(code=200, text='Success') response_dict = response.to_dict() self.assertEqual('ftp://foofle.com', response_dict['request']['url']) self.assertEqual('ftp', response_dict['protocol']) self.assertEqual(200, response_dict['reply']['code']) self.assertEqual(200, response_dict['response_code']) self.assertEqual('Success', response_dict['reply']['text']) self.assertEqual('Success', response_dict['response_message'])
def test_to_dict_body(self): response = Response() response.body = Body() response_dict = response.to_dict() self.assertTrue(response_dict['body']) response.body.close() response = Response() response.body = NotImplemented response_dict = response.to_dict() self.assertFalse(response_dict['body'])
def test_warc_recorder_ftp(self): file_prefix = 'asdf' warc_filename = 'asdf.warc' warc_recorder = WARCRecorder(file_prefix, params=WARCRecorderParams(compress=False)) request = FTPRequest('ftp://example.com/example.txt') request.address = ('0.0.0.0', 80) response = FTPResponse() response.reply = FTPReply(200, 'OK') response.body = Body() response.data_address = ('0.0.0.0', 12345) with wpull.util.reset_file_offset(response.body): response.body.write(b'KITTEH DOGE') with warc_recorder.session() as session: session.begin_control(request) session.request_control_data(b'GIMMEH example.txt') session.response_control_data(b'200 OK, no need to yell.') session.pre_response(response) session.response_data(b'KITTEH DOGE') session.response(response) session.end_control(response) warc_recorder.close() with open(warc_filename, 'rb') as in_file: warc_file_content = in_file.read() self.assertTrue(warc_file_content.startswith(b'WARC/1.0')) self.assertIn(b'WARC-Type: warcinfo\r\n', warc_file_content) self.assertIn(b'Content-Type: application/warc-fields', warc_file_content) self.assertIn(b'WARC-Date: ', warc_file_content) self.assertIn(b'WARC-Record-ID: <urn:uuid:', warc_file_content) self.assertIn(b'WARC-Block-Digest: sha1:', warc_file_content) self.assertNotIn(b'WARC-Payload-Digest: sha1:', warc_file_content) self.assertIn(b'WARC-Type: resource\r\n', warc_file_content) self.assertIn(b'WARC-Target-URI: ftp://', warc_file_content) self.assertIn(b'Content-Type: application/octet-stream', warc_file_content) self.assertIn(b'WARC-Type: metadata', warc_file_content) self.assertIn(b'WARC-Concurrent-To: <urn:uuid:', warc_file_content) self.assertIn(b'Content-Type: text/x-ftp-control-conversation', warc_file_content) self.assertIn( 'Wpull/{0}'.format(wpull.version.__version__).encode('utf-8'), warc_file_content) self.assertIn( 'Python/{0}'.format(wpull.util.python_version()).encode('utf-8'), warc_file_content) self.assertIn(b'KITTEH DOGE', warc_file_content) self.assertIn(b'* Opening control connection to', warc_file_content) self.assertIn(b'* Kept control connection to', warc_file_content) self.assertIn(b'* Opened data connection to ', warc_file_content) self.assertIn(b'* Closed data connection to ', warc_file_content) self.assertIn(b'> GIMMEH example.txt', warc_file_content) self.assertIn(b'< 200 OK, no need to yell.', warc_file_content) # Ignore Concurrent Record ID not seen yet self.validate_warc(warc_filename, ignore_minor_error=True) with open(warc_filename, 'r+b') as in_file: # Intentionally modify the contents in_file.seek(355) in_file.write(b'f') with self.assertRaises(Exception): # Sanity check that it actually raises error on bad digest self.validate_warc(warc_filename, ignore_minor_error=True)
def test_warc_recorder_ftp(self): file_prefix = 'asdf' warc_filename = 'asdf.warc' warc_recorder = WARCRecorder( file_prefix, params=WARCRecorderParams(compress=False) ) request = FTPRequest('ftp://example.com/example.txt') request.address = ('0.0.0.0', 80) response = FTPResponse() response.reply = FTPReply(200, 'OK') response.body = Body() response.data_address = ('0.0.0.0', 12345) with wpull.util.reset_file_offset(response.body): response.body.write(b'KITTEH DOGE') with warc_recorder.session() as session: session.begin_control(request) session.request_control_data(b'GIMMEH example.txt') session.response_control_data(b'200 OK, no need to yell.') session.pre_response(response) session.response_data(b'KITTEH DOGE') session.response(response) session.end_control(response) warc_recorder.close() with open(warc_filename, 'rb') as in_file: warc_file_content = in_file.read() self.assertTrue(warc_file_content.startswith(b'WARC/1.0')) self.assertIn(b'WARC-Type: warcinfo\r\n', warc_file_content) self.assertIn(b'Content-Type: application/warc-fields', warc_file_content) self.assertIn(b'WARC-Date: ', warc_file_content) self.assertIn(b'WARC-Record-ID: <urn:uuid:', warc_file_content) self.assertIn(b'WARC-Block-Digest: sha1:', warc_file_content) self.assertNotIn(b'WARC-Payload-Digest: sha1:', warc_file_content) self.assertIn(b'WARC-Type: resource\r\n', warc_file_content) self.assertIn(b'WARC-Target-URI: ftp://', warc_file_content) self.assertIn(b'Content-Type: application/octet-stream', warc_file_content) self.assertIn(b'WARC-Type: metadata', warc_file_content) self.assertIn(b'WARC-Concurrent-To: <urn:uuid:', warc_file_content) self.assertIn(b'Content-Type: text/x-ftp-control-conversation', warc_file_content) self.assertIn( 'Wpull/{0}'.format(wpull.version.__version__).encode('utf-8'), warc_file_content ) self.assertIn( 'Python/{0}'.format( wpull.util.python_version()).encode('utf-8'), warc_file_content ) self.assertIn(b'KITTEH DOGE', warc_file_content) self.assertIn(b'* Opening control connection to', warc_file_content) self.assertIn(b'* Kept control connection to', warc_file_content) self.assertIn(b'* Opened data connection to ', warc_file_content) self.assertIn(b'* Closed data connection to ', warc_file_content) self.assertIn(b'> GIMMEH example.txt', warc_file_content) self.assertIn(b'< 200 OK, no need to yell.', warc_file_content) # Ignore Concurrent Record ID not seen yet self.validate_warc(warc_filename, ignore_minor_error=True) with open(warc_filename, 'r+b') as in_file: # Intentionally modify the contents in_file.seek(355) in_file.write(b'f') with self.assertRaises(Exception): # Sanity check that it actually raises error on bad digest self.validate_warc(warc_filename, ignore_minor_error=True)