def _test_command(self, curl_command, expected_result): result = curl_to_request_kwargs(curl_command) self.assertEqual(result, expected_result) try: Request(**result) except TypeError as e: self.fail("Request kwargs are not correct {}".format(e))
def test_too_few_arguments_error(self): assertRaisesRegex( self, ValueError, r"too few arguments|the following arguments are required:\s*url", lambda: curl_to_request_kwargs("curl"), )
def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): """Create a Request object from a string containing a `cURL <https://curl.haxx.se/>`_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same arguments as the :class:`Request` class, taking preference and overriding the values of the same arguments contained in the cURL command. Unrecognized options are ignored by default. To raise an error when finding unknown options call this method by passing ``ignore_unknown_options=False``. .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` subclasses, such as :class:`~scrapy.http.JSONRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares <topics-downloader-middleware>` and :ref:`spider middlewares <topics-spider-middleware>` enabled, such as :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`, :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`, or :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, may modify the :class:`~scrapy.http.Request` object. """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs)
def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: with warnings.catch_warnings(): # avoid warning when executing tests warnings.simplefilter('ignore') curl_command = 'curl --bar --baz http://www.example.com' expected_result = {"method": "GET", "url": "http://www.example.com"} self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): self.assertRaisesRegex( ValueError, "Unrecognized options:.*--bar.*--baz", lambda: curl_to_request_kwargs( "curl --bar --baz http://www.example.com", ignore_unknown_options=False ), )
def test_must_start_with_curl_error(self): self.assertRaises( ValueError, lambda: curl_to_request_kwargs("carl -X POST http://example.org") )
def test_get_silent(self): curl_command = 'curl --silent "www.example.com"' expected_result = {"method": "GET", "url": "http://www.example.com"} self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)