def download_request(self, request, spider):
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse) and
uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)
def download_request(self, request, spider):
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse) and
uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)
def download_request(self, request, spider):
# XXX: I think this needs urllib.unquote().
m = self._data_url_pattern.match(request.url)
if not m:
raise ValueError("invalid data URL")
mimetype, is_base64, data = m.groups()
data = urllib.unquote(data)
if is_base64:
data = base64.urlsafe_b64decode(data)
respcls = responsetypes.from_mimetype(mimetype)
return respcls(url=request.url, body=data)
def download_request(self, request, spider):
# XXX: I think this needs urllib.unquote().
m = self._data_url_pattern.match(request.url)
if not m:
raise ValueError("invalid data URL")
mimetype, is_base64, data = m.groups()
data = urllib.unquote(data)
if is_base64:
data = base64.b64decode(data)
respcls = responsetypes.from_mimetype(mimetype)
return respcls(url=request.url, body=data)
def download_request(self, request, spider):
if self.download_opts_field in request.meta:
opts = request.meta[self.download_opts_field]
download_opts = merge(self.default_opts, opts)
else:
download_opts = self.default_opts
download_url = request.url
download_path = self.download_func(download_url, download_opts)
respcls = responsetypes.from_mimetype('text/*')
response = respcls(request=request,
url=download_url,
encoding='utf-8',
body=download_path)
return response
def _test(response):
self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "utf-8")
def _test(response):
self.assertEqual(response.text, 'A brief note')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "US-ASCII")
def _test(response):
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "utf-8")
def _test(response):
self.assertEqual(response.text, 'A brief note')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "US-ASCII")
def _test(response):
self.assertEquals(response.text, u'\u038e\u03a3\u038e')
self.assertEquals(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEquals(response.encoding, "iso-8859-7")