def test_adapter_performs_a_successful_request_through_proxy():
request = PreparedRequest()
request.prepare(url="http://somefakeurl", method="GET", headers={})
proxies = {
"http": "http://localhost:8080",
"https": "https://localhost:8081",
}
header_lines = [
b"HTTP/1.1 200 OK\n",
b"Content-Language: en-US\n",
]
pool = FakePool()
pool.add_response(200, b"data obtained through proxy", header_lines)
pool_provider = FakePoolProvider()
pool_provider.add_pool_for_proxied_url("http://localhost:8080",
request.url, pool)
adapter = CURLAdapter(
pool_provider_factory=lambda *args, **kwargs: pool_provider)
response = adapter.send(request, proxies=proxies)
assert response.status_code == 200
assert response.text == "data obtained through proxy"
assert response.headers == {"Content-Language": "en-US"}
def main(query, pages, city):
city_url = urljoin(BASE_URL, city + '/')
url = urljoin(city_url, f'q-{query}')
response = requests.get(url)
soup = get_soup(response)
electronics_tag = soup.find('a', attrs={'data-id': '37'})
electronics_link = electronics_tag.attrs['href']
response = requests.get(electronics_link)
soup = get_soup(response)
accessories_tag = soup.find('a', attrs={'data-id': '44'})
accessories_link = accessories_tag.attrs['href']
response = requests.get(accessories_link)
soup = get_soup(response)
phones_tag = soup.find('a', attrs={'data-id': '85'})
phones_link = phones_tag.attrs['href']
links = []
req = PreparedRequest()
for page_number in range(pages):
req.prepare_url(phones_link, params={'page': page_number + 1})
page_links = get_links_on_page(req.url)
links.extend(page_links)
filename = f'{query}.csv'
write_to_csv_file(filename, links)
def test_get_cutout_success(self, mock_session):
resolution = 0
x_range = [20, 40]
y_range = [50, 70]
z_range = [30, 50]
time_range = [10, 25]
id_list = []
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
fake_prepped_req = PreparedRequest()
fake_prepped_req.headers = {}
mock_session.prepare_request.return_value = fake_prepped_req
data = numpy.random.randint(0, 3000, (15, 20, 20, 20), numpy.uint16)
compressed_data = blosc.compress(data, typesize=16)
fake_response = Response()
fake_response.status_code = 200
fake_response._content = compressed_data
mock_session.send.return_value = fake_response
send_opts = {}
actual = self.vol.get_cutout(self.chan, resolution, x_range, y_range,
z_range, time_range, id_list, url_prefix,
auth, mock_session, send_opts)
numpy.testing.assert_array_equal(data, actual)
def test_prj_create_success(self, mock_session, mock_resp):
mock_session.prepare_request.return_value = PreparedRequest()
mock_resp.status_code = 201
mock_resp.json.return_value = {
'type': 'image',
'name': 'chan',
'description': 'walker',
'experiment': 'bar',
'creator': 'me',
'default_time_sample': 2,
'datatype': 'uint16',
'base_resolution': 0
}
mock_session.send.return_value = mock_resp
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
send_opts = {}
actual = self.prj.create(self.chan, url_prefix, auth, mock_session,
send_opts)
self.assertTrue(isinstance(actual, ChannelResource))
self.assertEqual('chan', actual.name)
self.assertEqual('foo', actual.coll_name)
self.assertEqual('bar', actual.exp_name)
self.assertEqual('image', actual.type)
def get_lookup_result(self, threat, atom_type, hashkey_only) -> list:
params = {'atom_value': threat, 'atom_type': atom_type, 'hashkey_only': hashkey_only}
req = PreparedRequest() # Adding parameters using requests' tool
req.prepare_url(self.url, params)
response = self.datalake_requests(req.url, 'get',
headers={'Authorization': self.tokens[0]})
return response
def test_get_ids_in_region_success(self, mock_session, mock_resp):
resolution = 0
x_range = [0, 100]
y_range = [10, 50]
z_range = [20, 42]
t_range = [0, 1]
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
send_opts = {}
fake_prepped_req = PreparedRequest()
fake_prepped_req.headers = {}
mock_session.prepare_request.return_value = fake_prepped_req
mock_session.send.return_value = mock_resp
mock_resp.status_code = 200
mock_resp.json.return_value = {'ids': ['1', '10']}
actual = self.vol.get_ids_in_region(self.anno_chan, resolution,
x_range, y_range, z_range, t_range,
url_prefix, auth, mock_session,
send_opts)
expected = [1, 10]
self.assertEqual(expected, actual)
def test__call__(mocker):
auth = SpotifyAPIAuth()
mocker.patch.object(auth, "_set_tokens")
auth.tokens = {"access_token": "xxxxx"}
req = PreparedRequest()
req.prepare(method="GET", url="https://spotify.com", auth=auth)
assert req.headers["authorization"] == "Bearer xxxxx"
def test_token(self):
token = TokenAuth('tokenvalue')
req = PreparedRequest()
req.url = 'http://test.com/'
req.method = 'GET'
req.prepare_auth(token)
self.assertEqual(req.url, 'http://test.com/?token=tokenvalue')
def make_url(api_key, location_name):
"""
Arguments:
api_key (String)
The Mapzen API key you get during registration
e.g. 'MY_API_KEY'
location_name (String)
Represents the human-readable name of the location to geocode
e.g. 'Stanford University'
Returns:
String
A url formatted according to Mapzen Search API spec.
e.g. https://search.mapzen.com/v1/search?api_key=MY_API_KEY&text=Stanford+University
"""
my_params = {'text': location_name, 'api_key': api_key}
p = PreparedRequest()
p.prepare_url(url=BASE_ENDPOINT, params=my_params)
return p.url
def _parse_raw_request(cls, raw_request: bytes) -> PreparedRequest:
"""
ok, this is kind of janky, but AWS4Auth is meant to work with requests, so expects a PreparedRequest
"""
body: Optional[str] = None
headers, body = raw_request.decode('utf-8').split('\r\n\r\n', 1)
# strip the trailing \r\n if present
if len(body) == 0:
body = None
elif body.endswith('\r\n'):
body = body[:-2]
# hi! if you get here looking for folded headers, that's obsolete and we ought not be generating them
method_et_al, headers = headers.split('\r\n', 1)
headers_as_dict: Mapping[str, str] = \
dict([(k.strip(), v.strip()) for k, v in [h.split(':', 1) for h in headers.split('\r\n')]])
# this is a little janky, really should be one or more spaces
method, path_et_al, version = method_et_al.split(' ', 2)
# this is very sketchy looking but I promise that we don't care about the host, port, or scheme here
url = 'https://nope/' + path_et_al
req = PreparedRequest()
req.prepare_method(method)
req.prepare_url(url, {})
req.prepare_headers(headers_as_dict)
req.prepare_body(data=body, files=None)
# don't req.prepare_content_length, we already had that in headers surely
return req
def test_prj_update_success(self, mock_session, mock_resp):
chan_dict = {
'name': 'bar', 'description': 'none', 'experiment': 8, 'type': 'image',
'is_channel': True, 'default_time_sample': 0, 'datatype': 'uint16',
'base_resolution': 0, 'linked_channel_layers': [], 'creator': 'me'
}
expected = ChannelResource(
chan_dict['name'], self.chan.coll_name, self.chan.exp_name, 'image')
expected.description = chan_dict['description']
expected.datatype = chan_dict['datatype']
expected.base_resolution = chan_dict['base_resolution']
expected.default_time_sample = chan_dict['default_time_sample']
expected.type = chan_dict['type']
mock_session.prepare_request.return_value = PreparedRequest()
mock_resp.json.return_value = chan_dict
mock_resp.status_code = 200
mock_session.send.return_value = mock_resp
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
send_opts = {}
actual = self.prj.update(self.chan.name, self.chan, url_prefix, auth, mock_session, send_opts)
self.assertEqual(expected.name, actual.name)
self.assertEqual(expected.description, actual.description)
self.assertEqual(expected.exp_name, actual.exp_name)
self.assertEqual(expected.coll_name, actual.coll_name)
self.assertEqual(expected.default_time_sample, actual.default_time_sample)
self.assertEqual(expected.datatype, actual.datatype)
self.assertEqual(expected.base_resolution, actual.base_resolution)
self.assertEqual(expected.type, actual.type)
def test_pool_send_configures_handler_correctly():
prepared_request = PreparedRequest()
prepared_request.prepare(url="http://somefakeurl",
method="GET",
headers={})
curl_request = CURLRequest(prepared_request)
curl_handler = FakeCurlHandler()
# Configure the handler to return some body and headers
curl_handler.body = b"somebodydata"
curl_handler.http_status = 200
curl_handler.header_lines = [
"HTTP/1.1 200 OK\n".encode("iso-8859-1"),
"Content-Language: en-US\n".encode("iso-8859-1"),
"Cache-Control: no-cache\n".encode("iso-8859-1"),
]
pool = CURLHandlerPool(curl_factory=lambda: curl_handler)
response = pool.send(curl_request)
assert curl_handler.performed
assert response.body.getvalue() == b"somebodydata"
assert response.http_code == 200
assert response.headers == {
"Cache-Control": "no-cache",
"Content-Language": "en-US",
}
# Assert that the curl options from the requests were set to the handler
for opt, val in curl_request.options.items():
assert curl_handler.options[opt] == val
def test_pre_send():
"""Test the pre_send method."""
# backup time methods
backup_sleep = time.sleep
backup_time = time.time
rate_limit_handler = RateLimitHandler()
sleep = MagicMock(return_value=None)
time.sleep = sleep
time.time = MagicMock(return_value=7)
response: Response = Response()
type(response).ok = PropertyMock(True)
response.headers = {
'X-RateLimit-Remaining': 0,
'X-RateLimit-Reset': 10
}
rate_limit_handler.post_send(response)
request: PreparedRequest = PreparedRequest()
rate_limit_handler.pre_send(request)
sleep.assert_called_once_with(3)
# restore time methods
time.sleep = backup_sleep
time.time = backup_time
def test_curl_options_for_post_with_some_file_data(tmpdir):
p = tmpdir.join("test.txt")
p.write("content")
prepared_request = PreparedRequest()
prepared_request.prepare(
url="http://somefakeurl",
method="POST",
data=p,
)
curl_request = CURLRequest(prepared_request)
curl_options = curl_request.options
expected_headers = []
assert len(curl_options) == 7
assert curl_options[pycurl.URL] == "http://somefakeurl/"
assert curl_options[pycurl.SSL_VERIFYHOST] == 0
assert curl_options[pycurl.SSL_VERIFYPEER] == 0
assert curl_options[pycurl.UPLOAD] is True
assert curl_options[pycurl.CUSTOMREQUEST] == "POST"
assert curl_options[pycurl.HTTPHEADER] == expected_headers
# We actually call the function to test that it reads
# the expected bytes
assert curl_options[pycurl.READFUNCTION]() == "content"
def test_curl_response_with_cookies():
prepared_request = PreparedRequest()
prepared_request.prepare(url="http://somefakeurl",
method="GET",
headers={})
curl_request = CURLRequest(prepared_request)
curl_response = CURLResponse(curl_request)
curl_response.http_code = 200
header_lines = [
"HTTP/1.1 200 OK\n",
"Content-Language: en-US\n",
"Cache-Control: no-cache\n",
"Set-Cookie: foo=123; SameSite=None; Secure; Max-Age=2592000\n",
"Set-Cookie: bar=abc; HttpOnly\n",
]
for header_line in header_lines:
# We provide lines encoded as defined in http standard
curl_response.add_header_from_raw_line(
header_line.encode("iso-8859-1"))
req_response = curl_response.to_requests_response()
assert len(req_response.cookies) == 2
assert req_response.cookies.get("foo") == "123"
assert req_response.cookies.get("bar") == "abc"
def test_curl_options_for_post_put_with_some_string_data(
http_method, data, expected_data):
prepared_request = PreparedRequest()
prepared_request.prepare(
url="http://somefakeurl",
method=http_method,
data=data,
)
curl_request = CURLRequest(prepared_request)
curl_options = curl_request.options
expected_headers = [
f"Content-Length: {len(data)}",
]
assert len(curl_options) == 7
assert curl_options[pycurl.URL] == "http://somefakeurl/"
assert curl_options[pycurl.SSL_VERIFYHOST] == 0
assert curl_options[pycurl.SSL_VERIFYPEER] == 0
assert curl_options[pycurl.UPLOAD] is True
assert curl_options[pycurl.CUSTOMREQUEST] == http_method
assert curl_options[pycurl.HTTPHEADER] == expected_headers
# We actually call the function to test that it reads
# the expected bytes
assert curl_options[pycurl.READFUNCTION]() == expected_data
def test_get_bounding_box_success(self, mock_session, mock_resp):
resolution = 0
id = 44444
bb_type = 'loose'
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
send_opts = {}
fake_prepped_req = PreparedRequest()
fake_prepped_req.headers = {}
mock_session.prepare_request.return_value = fake_prepped_req
mock_session.send.return_value = mock_resp
mock_resp.status_code = 200
mock_resp.json.return_value = expected = {
'x_range': [0, 10],
'y_range': [0, 10],
'z_range': [0, 10],
't_range': [0, 10]
}
actual = self.vol.get_bounding_box(self.anno_chan, resolution, id,
bb_type, url_prefix, auth,
mock_session, send_opts)
self.assertEqual(expected, actual)
def fetch(self, entry_url: str) -> JobsList:
self.jobs = JobsList()
page_buffer = []
for job_link in self.get_jobs_list(entry_url):
try:
page_buffer.append(self.get_job(job_link))
except Exception as e:
print("Error Processing %s %s " % (job_link, e))
page = 1
while len(page_buffer) > 0:
self.jobs.extend(page_buffer)
page_buffer = []
prep_url = PreparedRequest()
prep_url.prepare(url=entry_url, params={'page': page})
next_page_url = prep_url.url
for job_link in self.get_jobs_list(next_page_url):
try:
page_buffer.append(self.get_job(job_link))
except Exception as e:
print("Error Processing %s %s " % (job_link, e))
print("Scraped page %s" % page)
page += 1
return self.jobs
def get_threats(self, query_hash: str, limit=10, response_format="application/json") -> dict:
url = self.url + query_hash
params = {'limit': limit}
req = PreparedRequest() # Adding parameters using requests' tool
req.prepare_url(url, params)
headers = {'Authorization': self.tokens[0], 'Accept': response_format}
response = self.datalake_requests(req.url, 'get', headers=headers)
return response
def get_query(self, data=None):
p = PreparedRequest()
req_data = build_req_data(data)
p.prepare_url(
self.target,
req_data) # 该方法只是其中一个方法，这个方法就是专门拼接params与url的，生成 EncodeUrl。
self.query = p.url
return self.query
def sub_link(self):
'''生成该用户的订阅地址'''
p = PreparedRequest()
token = base64.b64encode(self.username.encode()).decode()
url = settings.HOST + 'server/subscribe/'
params = {'token': token}
p.prepare_url(url, params)
return p.url
def sub_link(self):
"""生成该用户的订阅地址"""
p = PreparedRequest()
token = base64.b64encode(self.username.encode()).decode()
url = settings.HOST + "/server/subscribe/"
params = {"token": token}
p.prepare_url(url, params)
return p.url
def test_http_method(self):
unknown_method = ZoomAPIException(0, 'Test', None, 'Test message')
self.assertEqual(unknown_method.http_method, None)
req = PreparedRequest()
req.prepare_method('GET')
known_method = ZoomAPIException(0, 'Test', req, 'Message')
self.assertEqual(known_method.http_method, 'GET')
def _add_url_params(self, url: str, extra_params: dict):
"""Add params to URL."""
params = self.params.copy()
params.update(extra_params)
req = PreparedRequest()
req.prepare_url(url, params)
return req.url
def setUp(self):
self.alt_url = ('http://example.com/path/to/end/point?query=string'
'&foo=bar')
self.p = PreparedRequest()
self.p.body = 'Foo bar'
self.p.headers = {'User-Agent': 'betamax/test'}
self.p.url = 'http://example.com/path/to/end/point?query=string'
self.p.method = 'GET'
self.p._cookies = RequestsCookieJar()
def test_use_chunked_upload_is_false_for_json():
prepared_request = PreparedRequest()
json_data = {"somekey": "somedata"}
prepared_request.prepare(
url="http://somefakeurl",
json=json_data,
)
curl_request = CURLRequest(prepared_request)
assert curl_request.use_chunked_upload is False
def test_use_chunked_upload_is_false_for_string_data():
prepared_request = PreparedRequest()
some_data = "this is some data as string"
prepared_request.prepare(
url="http://somefakeurl",
data=some_data,
)
curl_request = CURLRequest(prepared_request)
assert curl_request.use_chunked_upload is False
def test_use_chunked_upload_is_true_for_streamed_data():
prepared_request = PreparedRequest()
some_stream_data = six.StringIO("this is some data as string")
prepared_request.prepare(
url="http://somefakeurl",
data=some_stream_data,
)
curl_request = CURLRequest(prepared_request)
assert curl_request.use_chunked_upload is True
def test_token_auth(self):
token = {'access_token': '123456789'}
auth = OAuthTokenAuthentication("client_id", token)
session = auth.signed_session()
request = PreparedRequest()
request.prepare("GET", "https://example.org")
session.auth(request)
assert request.headers == {'Authorization': 'Bearer 123456789'}
def test_delete_success(self, mock_session):
mock_session.prepare_request.return_value = PreparedRequest()
fake_resp = Response()
fake_resp.status_code = 204
mock_session.send.return_value = fake_resp
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
send_opts = {}
self.prj.delete_user('johndoe', url_prefix, auth, mock_session,
send_opts)
