def get_auth_url(self, client_id, state=None):
""" Get Hubspot OAuth Authorization URL """
params = {
'client_id': client_id,
'redirect_uri': SETTINGS['OAUTH_REDIRECT_URL'],
'scope': SETTINGS['SCOPE']
}
if state:
params['state'] = state
request = requests.PreparedRequest()
request.prepare_url(SETTINGS['OAUTH_AUTHORIZE_URL'], params=params)
return request.url
async def perform_request(self,
method,
url,
params=None,
body=None,
timeout=None,
ignore=(),
headers=None):
req = requests.PreparedRequest()
req.prepare(method, self.host + url, headers, None, body, params)
self.aws_auth(req) # sign the request
headers.update(req.headers)
return await super().perform_request(method, url, params, body,
timeout, ignore, headers)
def build_request(
self, method: str, suffix: str,
headers: typing.Union[None, typing.Dict[str, str]],
body: typing.Union[None, bytes]) -> requests.PreparedRequest:
request = requests.PreparedRequest()
request.method = method
request.data = body
request.headers = {}
if headers is not None:
request.headers = headers
request.headers["Authorization"] = "Bearer {}".format(
self._session.tokens().get("playlist-read"))
request.url = self._baseUrl + suffix
return request
def test_login(self, username, password):
resp = LoginTest.table.get("get_cookie")
send_data = {
"user_name": username,
"user_password": password,
"login_security": resp.filter(LoginTest.tools.get("security")),
"http_referer": "/DBshop"
}
p = requests.PreparedRequest()
p.prepare(url="http://192.168.1.16/DBshop/user/login",
method="post",
headers={"Cookie": resp.get_header_by("set-cookie")},
data=send_data)
response = LoginTest.client.do(p)
self.assertTrue(username in response.text)
def test_multiple_token_authenticator():
multiple_token_auth = MultipleTokenAuthenticator(
tokens=["token1", "token2"])
header1 = multiple_token_auth.get_auth_header()
header2 = multiple_token_auth.get_auth_header()
header3 = multiple_token_auth.get_auth_header()
prepared_request = requests.PreparedRequest()
prepared_request.headers = {}
multiple_token_auth(prepared_request)
assert {"Authorization": "Bearer token2"} == prepared_request.headers
assert {"Authorization": "Bearer token1"} == header1
assert {"Authorization": "Bearer token2"} == header2
assert {"Authorization": "Bearer token1"} == header3
def fake_response(self, method, path, payload, code, content):
request = requests.PreparedRequest()
request.method = method
request.url = self.proxy.url(path)
request.headers = {'Content-Type': 'application/json'}
request.body = None
if method in ['get', 'delete']:
request.params = payload
elif method in ['put', 'post']:
request.data = json.dumps(payload)
response = requests.Response()
response.request = request
response.status_code = code
response._content = json.dumps(content) if content else ''
return response
def request_body_read_callback(chunk: bytes):
should_pipe_to_stdout = bool(
# Request body output desired
OUT_REQ_BODY in args.output_options
# & not `.read()` already pre-request (e.g., for compression)
and initial_request
# & non-EOF chunk
and chunk
)
if should_pipe_to_stdout:
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = chunk
msg.headers = initial_request.headers
write_message(requests_message=msg, env=env, args=args, with_body=True, with_headers=False)
def test_reports_stream_send_request_backoff_exception(mocker, caplog,
reports_stream):
response = requests.Response()
response.status_code = 429
mocker.patch.object(requests.Session, "send", return_value=response)
mocker.patch.object(time, "sleep", return_value=None)
with pytest.raises(DefaultBackoffException):
reports_stream._send_request(request=requests.PreparedRequest())
assert "Backing off _send_request(...) for 5.0s" in caplog.text
assert "Backing off _send_request(...) for 10.0s" in caplog.text
assert "Backing off _send_request(...) for 20.0s" in caplog.text
assert "Backing off _send_request(...) for 40.0s" in caplog.text
assert "Giving up _send_request(...) after 5 tries" in caplog.text
def list_mines(clt: 'SketchFabClient',
sort_by: str = None,
downloadable: bool = None,
collection: SFCollection = None) -> List[SFModel]:
# Using a prepared request is the simplest way to build an URL
params = SFModelsApi._list_prepare_params_common(
sort_by=sort_by,
downloadable=downloadable,
collection=collection,
)
req = requests.PreparedRequest()
req.prepare_url(f'{API_URL}/me/search', params)
data = clt.session.get(req.url).json()
return [SFModel(m, clt) for m in data['results']]
def test_token_authenticator():
"""
Should match passed in token, no matter how many times token is retrieved.
"""
token_auth = TokenAuthenticator(token="test-token")
header1 = token_auth.get_auth_header()
header2 = token_auth.get_auth_header()
prepared_request = requests.PreparedRequest()
prepared_request.headers = {}
token_auth(prepared_request)
assert {"Authorization": "Bearer test-token"} == prepared_request.headers
assert {"Authorization": "Bearer test-token"} == header1
assert {"Authorization": "Bearer test-token"} == header2
def test_fetch_page_error(self, mock_get_region, mock_proxies,
mock_headers, mock_requests):
"""Tests that fetch_page successfully handles error responses."""
url = "/around/the/world"
region = "us_sd"
initial_task = "work_it"
mock_get_region.return_value = mock_region(region)
proxies = {"http": "http://*****:*****@proxy.biz/"}
mock_proxies.return_value = proxies
headers = {"User-Agent": "test_user_agent"}
mock_headers.return_value = headers
original_request = requests.PreparedRequest()
original_request.headers = headers
original_request.method = "GET"
original_request.body = None
# test a few types of errors
errors = {
500: "SERVER ERROR",
502: "PROXY ERROR",
503: "SERVICE UNAVAILABLE",
}
error_response = requests.Response()
error_response.headers = {}
error_response.request = original_request
for code, name in errors.items():
error_response.status_code = code
error_response.reason = name
mock_requests.return_value = error_response
scraper = FakeScraper(region, initial_task)
with pytest.raises(FetchPageError):
scraper.fetch_page(url)
mock_get_region.assert_called_with(region)
mock_proxies.assert_called_with()
mock_headers.assert_called_with()
mock_requests.assert_called_with(
url,
proxies=proxies,
headers=headers,
cookies=None,
params=None,
verify=False,
)
def make_url(self, container=None, resource=None, query_items=None):
"""Create a URL from the specified parts."""
pth = [self._base_url]
if container:
pth.append(container.strip('/'))
if resource:
pth.append(resource)
else:
pth.append('')
url = '/'.join(pth)
if isinstance(query_items, (list, tuple, set)):
url += RestHttp._list_query_str(query_items)
query_items = None
p = requests.PreparedRequest()
p.prepare_url(url, query_items)
return p.url
def make_map_url(locations):
"""
assume locations is a list of dicts, with 'latitude' and 'longitude'
keys
"""
myparams = {}
myparams['size'] = MAP_SIZE
myparams['markers'] = []
for loc in locations:
marker = loc['latitude'] + ',' + loc['longitude']
myparams['markers'].append(marker)
pre_req = requests.PreparedRequest()
pre_req.prepare_url(BASE_MAP_URL, myparams)
return pre_req.url
def _build_url(self, url, params):
if not self._base_url:
raise ImproperlyConfigured("{} has no URL configured".format(self.__class__.__name__))
url = urlparse.urljoin(self._base_url, url)
# Make sure we always preserve the base_url host and scheme
# eg when using next link from the API response we need to keep the scheme
# so that app requests don't try to switch from HTTP to HTTPS
url = urlparse.urlparse(url)
base_url = urlparse.urlparse(self._base_url)
url = url._replace(netloc=base_url.netloc, scheme=base_url.scheme).geturl()
r = requests.PreparedRequest()
r.prepare_url(url=url, params=params)
return r.url
def test_session_send(requests_mock):
shutil.rmtree(urlquick.CACHE_LOCATION, ignore_errors=True)
url = 'https://www.test.com/1'
mocked = requests_mock.get(url, body=b"data")
session = urlquick.Session()
# Build Request object
req = requests.PreparedRequest()
req.prepare_method("GET")
req.prepare_url(url, None)
req.prepare_headers(None)
req.prepare_body(b"", None, None)
ret = session.send(req)
assert mocked.called
assert ret.content == b"data"
def test_auth_call_method(self, mocker):
oauth = Oauth2Authenticator(
token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint,
client_id=TestOauth2Authenticator.client_id,
client_secret=TestOauth2Authenticator.client_secret,
refresh_token=TestOauth2Authenticator.refresh_token,
)
mocker.patch.object(Oauth2Authenticator,
"refresh_access_token",
return_value=("access_token", 1000))
prepared_request = requests.PreparedRequest()
prepared_request.headers = {}
oauth(prepared_request)
assert {
"Authorization": "Bearer access_token"
} == prepared_request.headers
def get_previous_bit_price(unix_start_date):
# Dates must be formated in a year/month/day format
# Could break up this part onto its own function s
formated_start_date = datetime.utcfromtimestamp(unix_start_date).strftime(
'%Y-%m-%d')
formated_end_date = datetime.utcfromtimestamp(
unix_start_date - seconds_in_month).strftime('%Y-%m-%d')
print(formated_start_date, formated_end_date)
req = requests.PreparedRequest()
req.prepare_url(price_history_url, {
"start": formated_end_date,
"end": formated_start_date
})
print(req.url)
bitcoin_price = requests.get(req.url)
print(bitcoin_price.json()['bpi'])
return bitcoin_price.json()['bpi'][formated_start_date]
def generate_auth_url(scopes: List[str]):
global all_scopes
for scope in scopes:
if scope not in all_scopes:
raise RuntimeError("Not sure that's a valid scope")
pr = requests.PreparedRequest()
pr.prepare_url(
url="https://accounts.spotify.com/authorize",
params={
"client_id": "6306b3af252b4b2c8a55c1db34c5da95",
"response_type": "token",
"redirect_uri": "https://example.com",
"scope": " ".join(scopes),
},
)
return pr.url
def authorize_uri(self, redirect_uri, state=''):
'''
TODO: why i need this?
The authorization code is obtained by using an authorization server as an
intermediary between the client and resource owner.
Returns link to a login page location.
Web applications are higly advised to use the Proof Key for
Code Exchange scheme (PKCE) for security concerns.
'''
url = urlparse.urljoin(self.server, '/restapi/oauth/authorize')
params = {
'response_type': 'code',
'state': state,
'redirect_uri': redirect_uri,
'client_id': self.client_id
}
req = requests.PreparedRequest()
req.prepare_url(url, params=params)
return req.url
def write_raw_data(env: Environment,
data: Any,
*,
processing_options: Optional[ProcessingOptions] = None,
headers: Optional[HTTPHeadersDict] = None,
stream_kwargs: Optional[Dict[str, Any]] = None):
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = data
msg.headers = headers or HTTPHeadersDict()
msg_output_options = OutputOptions.from_message(msg,
body=True,
headers=False)
return write_message(requests_message=msg,
env=env,
output_options=msg_output_options,
processing_options=processing_options
or ProcessingOptions(),
extra_stream_kwargs=stream_kwargs)
def test_send_with_local_file_url(self, get_size_mock, open_mock):
transport = service.RequestsTransport()
url = 'file:///foo'
request = requests.PreparedRequest()
request.url = url
data = "Hello World"
get_size_mock.return_value = len(data)
def readinto_mock(buf):
buf[0:] = data
open_mock.return_value = mock.MagicMock(name='file_handle', spec=file)
file_handle = open_mock.return_value.__enter__.return_value
file_handle.readinto.side_effect = readinto_mock
resp = transport.session.send(request)
self.assertEqual(data, resp.content)
def _establish_kerberos(self,
url,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
parsed = urlparse(url)
crypt = None
try:
crypt = MSKerberosCrypt(parsed.hostname)
except GSSError:
crypt = MSKerberosCrypt(parsed.hostname, service="HTTP")
headers = {}
headers['Authorization'] = ("Kerberos " + crypt.get_token())
headers["Content-Type"] = "application/soap+xml;charset=UTF-8"
headers["Connection"] = 'Keep-Alive'
p = requests.PreparedRequest()
p.prepare_method("POST")
p.prepare_url(url, None)
p.prepare_headers(headers)
p.prepare_body("", None, None)
auth = HTTPMSKerberosAuth()
p.prepare_auth(auth, url)
verify = requests.adapters.HTTPAdapter.send(self, p, stream, timeout,
verify, cert, proxies)
field = verify.headers['www-authenticate']
kind, __, details = field.strip().partition(" ")
if kind.lower() == "kerberos":
crypt.step(details.strip())
HTTPMSKerberosAdapter.krb_dict[url] = crypt
verify.content
verify.close()
return verify
def make_url(api_key, location_name):
"""
Arguments:
api_key: String, the Mapzen API key you get during registration
e.g. 'search-blah'
location_name: String, represents the human-readable name of the location to geocode
e.g. 'Stanford University'
Returns:
String, a url formatted according to Mapzen Search API spec:
e.g. https://search.mapzen.com/v1/search?api_key=search-blah&text=Stanford+University
"""
my_params = {'text': location_name, 'api_key': api_key}
p = requests.PreparedRequest()
p.prepare_url(url=BASE_ENDPOINT, params=my_params)
return p.url
def __prepare_gprest_call(self,
requestURL,
params=None,
headers=None,
restType='GET',
body=None):
"""Returns Authorization type and GP headers
"""
if self.__serviceAccount.is_iam_enabled():
auth = None
iam_api_key_header = {
self.__AUTHORIZATION_HEADER_KEY:
str('API-KEY ' + self.__serviceAccount.get_api_key())
}
if not headers is None:
headers.update(iam_api_key_header)
else:
headers = iam_api_key_header
elif self.__auth == self.BASIC_AUTH:
auth = (self.__serviceAccount.get_user_id(),
self.__serviceAccount.get_password())
elif self.__auth == self.HMAC_AUTH:
auth = None
# need to prepare url by appending params to the end
# before creating the hmac headers
fakeRequest = requests.PreparedRequest()
fakeRequest.prepare_url(requestURL, params=params)
preparedUrl = fakeRequest.url
hmacHeaders = self.__get_gaas_hmac_headers(method=restType,
url=preparedUrl,
body=body)
if not headers is None:
headers.update(hmacHeaders)
else:
headers = hmacHeaders
return auth, headers
def make_data_file():
unix_starting_2016 = [1451606400]
price_data = {}
for num in range(0, 45):
unix_starting_2016.append(unix_starting_2016[num] + seconds_in_month)
for month in unix_starting_2016:
start_date = datetime.utcfromtimestamp(month).strftime('%Y-%m-%d')
end_date = datetime.utcfromtimestamp(
month + seconds_in_month).strftime('%Y-%m-%d')
req = requests.PreparedRequest()
req.prepare_url(price_history_url, {
"start": start_date,
"end": end_date
})
if len(price_data.keys()) == 0:
print("Got range from", start_date, end_date)
price_data = requests.get(req.url).json()['bpi']
else:
print("Got range from", start_date, end_date)
price_data = {**price_data, **requests.get(req.url).json()['bpi']}
with open('price_history.json', 'w') as outfile:
json.dump(price_data, outfile)
def get_page(api_key,
username,
page_number=1,
from_uts=0,
to_uts=10000000000,
limit=200):
url = "http://ws.audioscrobbler.com/2.0/"
params = {
"method": "user.getrecenttracks",
"format": "json",
"username": username,
"api_key": api_key,
"limit": limit,
"page": page_number,
"from": from_uts,
"to": to_uts,
}
request = requests.PreparedRequest()
request.prepare_url(url, params)
logger.info(
f"Requesting: {request.url.replace(api_key, 'REDACTED_API_KEY')}")
response = requests.get(request.url)
if response.status_code != 200:
logger.error(f"Non-200 Status: {response.status_code}")
response_json = response.json()
if "error" in response_json:
logger.error(
f"Error {response_json['error']}: {response_json['message']}")
return response_json
def prepare_request_for_twisted(request_params):
# type: (typing.MutableMapping[str, typing.Any]) -> typing.Mapping[str, typing.Any]
"""
Uses the python package 'requests' to prepare the data as per twisted
needs. requests.PreparedRequest.prepare is able to compute the body and
the headers for the http call based on the input request_params. This
contains any query parameters, files, body and headers to include.
:return: dictionary in the form
{
'body': string, # (can represent any content-type i.e. json,
file, multipart..),
'headers': dictionary, # headers->values
'method': string, # can be 'GET', 'POST' etc.
'url': string,
'timeout': float, # optional
'connect_timeout': float, # optional
}
"""
prepared_request = requests.PreparedRequest()
# Ensure that all the headers are converted to strings.
# This is need to workaround https://github.com/requests/requests/issues/3491
request_params['headers'] = {
k: v if isinstance(v, six.binary_type) else str(v)
for k, v in six.iteritems(request_params.get('headers', {}))
}
prepared_request.prepare(
headers=request_params.get('headers'),
data=request_params.get('data'),
params=request_params.get('params'),
files=request_params.get('files'),
url=request_params.get('url'),
method=request_params.get('method')
)
# content-length was computed by 'requests' based on the current body
# but body will be processed by fido using twisted FileBodyProducer
# causing content-length to lose meaning and break the client.
prepared_request.headers.pop('Content-Length', None)
request_for_twisted = {
# converting to string for `requests` method is necessary when
# using requests < 2.8.1 due to a bug while handling unicode values
# See changelog 2.8.1 at https://pypi.python.org/pypi/requests
'method': str(prepared_request.method or 'GET'),
'body': (
to_bytes(prepared_request.body)
if prepared_request.body is not None else None
),
'headers': prepared_request.headers,
'url': prepared_request.url,
}
for fetch_kwarg in ('connect_timeout', 'timeout', 'tcp_nodelay'):
if fetch_kwarg in request_params:
request_for_twisted[fetch_kwarg] = request_params[fetch_kwarg]
return request_for_twisted
import requests
import webbrowser
GMAPS_URL = 'https://maps.googleapis.com/maps/api/staticmap'
# get the latitude/longitude pairs
coordinate_pairs = ["12,77"]
# this is another way of serializing the URL
preq = requests.PreparedRequest()
preq.prepare_url(
GMAPS_URL, {
'size': '800x500',
'zoom': '13',
'markers': coordinate_pairs,
'key': 'AIzaSyBMJ6OmoVGaX2TLUu_GY083Kn9BiCqhKRk'
})
webbrowser.open(preq.url),
def test_reports_stream_send_request(mocker, reports_stream):
response = requests.Response()
response.status_code = 200
mocker.patch.object(requests.Session, "send", return_value=response)
assert response == reports_stream._send_request(request=requests.PreparedRequest())
def test_get_cookie(self):
p = requests.PreparedRequest()
p.prepare(url="http://192.168.1.16/DBshop/user/login", method="get")
LoginTest.table.set("get_cookie", LoginTest.client.do(p))