def processHeaders(self, response):
headers = response.headers
if Registrar.DEBUG_API:
Registrar.registerMessage("headers: %s" % str(headers))
if self.service.namespace == 'wp-api':
total_pages_key = 'X-WP-TotalPages'
total_items_key = 'X-WP-Total'
else:
total_pages_key = 'X-WC-TotalPages'
total_items_key = 'X-WC-Total'
if total_items_key in headers:
self.total_pages = int(headers.get(total_pages_key,''))
if total_pages_key in headers:
self.total_items = int(headers.get(total_items_key,''))
# if self.progressCounter is None:
# self.progressCounter = ProgressCounter(total=self.total_pages)
# self.stopNextIteration = True
prev_endpoint = self.next_endpoint
self.next_endpoint = None
for rel, link in response.links.items():
if rel == 'next' and link.get('url'):
next_response_url = link['url']
# if Registrar.DEBUG_API:
# Registrar.registerMessage('next_response_url: %s' % str(next_response_url))
self.next_page = int(UrlUtils.get_query_singular(next_response_url, 'page'))
if not self.next_page:
return
assert \
self.next_page <= self.total_pages, \
"next page (%s) should be lte total pages (%s)" \
% (str(self.next_page), str(self.total_pages))
self.next_endpoint = UrlUtils.set_query_singular(prev_endpoint,'page', self.next_page)
# if Registrar.DEBUG_API:
# Registrar.registerMessage('next_endpoint: %s' % str(self.next_endpoint))
if self.next_page:
self.offset = (self.limit * self.next_page) + 1
def test_url_get_query_dict_singular(self):
result = UrlUtils.get_query_dict_singular(self.test_url)
self.assertEquals(
result,
{
'filter[limit]': '2',
'oauth_nonce': 'c4f2920b0213c43f2e8d3d3333168ec4a22222d1',
'oauth_timestamp': '1481601370',
'oauth_consumer_key':
'ck_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
'oauth_signature_method': 'HMAC-SHA1',
'oauth_signature': '3ibOjMuhj6JGnI43BQZGniigHh8=',
'page': '2'
}
)
def test_query_string_endpoint_url(self):
query_string_api_params = dict(**self.api_params)
query_string_api_params.update(dict(query_string_auth=True))
api = API(
**query_string_api_params
)
endpoint_url = api.requester.endpoint_url(self.endpoint)
endpoint_url = api.auth.get_auth_url(endpoint_url, 'GET')
expected_endpoint_url = '%s?consumer_key=%s&consumer_secret=%s' % (self.endpoint, self.consumer_key, self.consumer_secret)
expected_endpoint_url = UrlUtils.join_components([self.base_url, self.api_name, self.api_ver, expected_endpoint_url])
self.assertEqual(
endpoint_url,
expected_endpoint_url
)
endpoint_url = api.requester.endpoint_url(self.endpoint)
endpoint_url = api.auth.get_auth_url(endpoint_url, 'GET')
def test_APIPutWithSimpleQuery(self):
wcapi = API(**self.api_params)
response = wcapi.get('products')
first_product = (response.json())[0]
# from pprint import pformat
# print "first product %s" % pformat(response.json())
original_title = first_product['name']
product_id = first_product['id']
nonce = str(random.random())
response = wcapi.put('products/%s?page=2&per_page=5' % (product_id), {"name":str(nonce)})
request_params = UrlUtils.get_query_dict_singular(response.request.url)
response_obj = response.json()
self.assertEqual(response_obj['name'], str(nonce))
self.assertEqual(request_params['per_page'], '5')
wcapi.put('products/%s' % (product_id), {"name":original_title})
def test_APIPutWithSimpleQuery(self):
wcapi = API(**self.api_params)
response = wcapi.get('products')
first_product = (response.json())['products'][0]
original_title = first_product['title']
product_id = first_product['id']
nonce = b"%f" % (random.random())
response = wcapi.put('products/%s?filter%%5Blimit%%5D=5' %
(product_id),
{"product": {"title": text_type(nonce)}})
request_params = UrlUtils.get_query_dict_singular(response.request.url)
response_obj = response.json()
self.assertEqual(response_obj['product']['title'], text_type(nonce))
self.assertEqual(request_params['filter[limit]'], text_type(5))
wcapi.put('products/%s' % (product_id),
{"product": {"title": original_title}})
def test_sorted_params(self):
# Example given in oauth.net:
oauthnet_example_sorted = [('a', '1'), ('c', 'hi%%20there'),
('f', '25'), ('f', '50'), ('f', 'a'),
('z', 'p'), ('z', 't')]
oauthnet_example = copy(oauthnet_example_sorted)
random.shuffle(oauthnet_example)
# oauthnet_example_sorted = [
# ('a', '1'),
# ('c', 'hi%%20there'),
# ('f', '25'),
# ('z', 'p'),
# ]
self.assertEqual(UrlUtils.sorted_params(oauthnet_example),
oauthnet_example_sorted)
def __init__(self, service, endpoint):
assert isinstance(service, WPAPI_Service)
self.service = service
self.next_endpoint = endpoint
self.prev_response = None
self.total_pages = None
self.total_items = None
# self.progressCounter = None
endpoint_queries = UrlUtils.get_query_dict_singular(endpoint)
# print "endpoint_queries:", endpoint_queries
self.next_page = None
if 'page' in endpoint_queries:
self.next_page = int(endpoint_queries['page'])
self.limit = 10
if 'fliter[limit]' in endpoint_queries:
self.limit = int(endpoint_queries['filter[limit]'])
# print "slave limit set to to ", self.limit
self.offset = None
if 'filter[offset]' in endpoint_queries:
self.offset = int(endpoint_queries['filter[offset]'])
def get_verifier(self, request_token=None, wp_user=None, wp_pass=None):
""" pretends to be a browser, uses the authorize auth link, submits user creds to WP login form to get
verifier string from access token """
if request_token is None:
request_token = self.request_token
assert request_token, "need a valid request_token for this step"
if wp_user is None and self.wp_user:
wp_user = self.wp_user
if wp_pass is None and self.wp_pass:
wp_pass = self.wp_pass
authorize_url = self.authentication['oauth1']['authorize']
authorize_url = UrlUtils.add_query(authorize_url, 'oauth_token',
request_token)
# we're using a different session from the usual API calls
# (I think the headers are incompatible?)
# self.requester.get(authorize_url)
authorize_session = requests.Session()
login_form_response = authorize_session.get(authorize_url)
try:
login_form_action, login_form_data = self.get_form_info(
login_form_response, 'loginform')
except AssertionError, e:
#try to parse error
login_form_soup = BeautifulSoup(login_form_response.text, 'lxml')
error = login_form_soup.select_one('div#login_error')
if error and "invalid token" in error.string.lower():
raise UserWarning("Invalid token: %s" % repr(request_token))
else:
raise UserWarning(
"could not parse login form. Site is misbehaving. Original error: %s " \
% str(e)
)
def endpoint_url(self, endpoint):
endpoint = StrUtils.decapitate(endpoint, '/')
return UrlUtils.join_components(
[self.url, self.api, self.api_version, endpoint])
def api_url(self):
return UrlUtils.join_components([self.url, self.api])
def get_verifier(self, request_token=None, wp_user=None, wp_pass=None):
""" pretends to be a browser, uses the authorize auth link, submits user creds to WP login form to get
verifier string from access token """
if request_token is None:
request_token = self.request_token
assert request_token, "need a valid request_token for this step"
if wp_user is None and self.wp_user:
wp_user = self.wp_user
if wp_pass is None and self.wp_pass:
wp_pass = self.wp_pass
authorize_url = self.authentication['oauth1']['authorize']
authorize_url = UrlUtils.add_query(authorize_url, 'oauth_token',
request_token)
# we're using a different session from the usual API calls
# (I think the headers are incompatible?)
# self.requester.get(authorize_url)
authorize_session = requests.Session()
login_form_response = authorize_session.get(authorize_url)
login_form_params = {
'username': wp_user,
'password': wp_pass,
'token': request_token
}
try:
login_form_action, login_form_data = self.get_form_info(
login_form_response, 'loginform')
except AssertionError as exc:
self.parse_login_form_error(login_form_response, exc,
**login_form_params)
for name, values in login_form_data.items():
if name == 'log':
login_form_data[name] = wp_user
elif name == 'pwd':
login_form_data[name] = wp_pass
else:
login_form_data[name] = values[0]
assert 'log' in login_form_data, 'input for user login did not appear on form'
assert 'pwd' in login_form_data, 'input for user password did not appear on form'
# print "submitting login form to %s : %s" % (login_form_action, str(login_form_data))
confirmation_response = authorize_session.post(login_form_action,
data=login_form_data,
allow_redirects=True)
try:
authorize_form_action, authorize_form_data = self.get_form_info(
confirmation_response, 'oauth1_authorize_form')
except AssertionError as exc:
self.parse_login_form_error(confirmation_response, exc,
**login_form_params)
for name, values in authorize_form_data.items():
if name == 'wp-submit':
assert \
'authorize' in values, \
"apparently no authorize button, only %s" % str(values)
authorize_form_data[name] = 'authorize'
else:
authorize_form_data[name] = values[0]
assert 'wp-submit' in login_form_data, 'authorize button did not appear on form'
final_response = authorize_session.post(authorize_form_action,
data=authorize_form_data,
allow_redirects=False)
assert \
final_response.status_code == 302, \
"was not redirected by authorize screen, was %d instead. something went wrong" \
% final_response.status_code
assert 'location' in final_response.headers, "redirect did not provide redirect location in header"
final_location = final_response.headers['location']
# At this point we can chose to follow the redirect if the user wants,
# or just parse the verifier out of the redirect url.
# open to suggestions if anyone has any :)
final_location_queries = parse_qs(urlparse(final_location).query)
assert \
'oauth_verifier' in final_location_queries, \
"oauth verifier not provided in final redirect: %s" % final_location
self._oauth_verifier = final_location_queries['oauth_verifier'][0]
return self._oauth_verifier
def get_signature_base_string(cls, method, params, url):
base_request_uri = quote(UrlUtils.substitute_query(url), "")
query_string = quote(cls.flatten_params(params), '~')
return "&".join([method, base_request_uri, query_string])
def test_flatten_params(self):
self.assertEqual(
UrlUtils.flatten_params(self.twitter_params_raw),
self.twitter_param_string
)
def test_url_del_query_singular(self):
result = UrlUtils.del_query_singular(self.test_url, 'filter[limit]')
expected = "http://ich.local:8888/woocommerce/wc-api/v3/products?oauth_consumer_key=ck_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX&oauth_nonce=c4f2920b0213c43f2e8d3d3333168ec4a22222d1&oauth_signature=3ibOjMuhj6JGnI43BQZGniigHh8%3D&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1481601370&page=2"
self.assertEqual(result, expected)
def api_ver_url_no_port(self):
return UrlUtils.remove_port(self.api_ver_url)
def is_ssl(self):
return UrlUtils.is_ssl(self.url)
def test_flatten_params(self):
self.assertEqual(
StrUtils.to_binary(UrlUtils.flatten_params(
self.twitter_params_raw)),
StrUtils.to_binary(self.twitter_param_string))
def next(self):
if Registrar.DEBUG_API:
Registrar.registerMessage('start')
if self.next_endpoint is None:
if Registrar.DEBUG_API:
Registrar.registerMessage('stopping due to no next endpoint')
raise StopIteration()
# get API response
try:
self.prev_response = self.service.get(self.next_endpoint)
except ReadTimeout as e:
# instead of processing this endoint, do the page product by product
if self.limit > 1:
new_limit = 1
if Registrar.DEBUG_API:
Registrar.registerMessage('reducing limit in %s' % self.next_endpoint)
self.next_endpoint = UrlUtils.set_query_singular(
self.next_endpoint,
'filter[limit]',
new_limit
)
self.next_endpoint = UrlUtils.del_query_singular(
self.next_endpoint,
'page'
)
if self.offset:
self.next_endpoint = UrlUtils.set_query_singular(
self.next_endpoint,
'filter[offset]',
self.offset
)
self.limit = new_limit
# endpoint_queries = parse_qs(urlparse(self.next_endpoint).query)
# endpoint_queries = dict([
# (key, value[0]) for key, value in endpoint_queries.items()
# ])
# endpoint_queries['filter[limit]'] = 1
# if self.next_page:
# endpoint_queries['page'] = 10 * self.next_page
# print "endpoint_queries: ", endpoint_queries
# self.next_endpoint = UrlUtils.substitute_query(
# self.next_endpoint,
# urlencode(endpoint_queries)
# )
if Registrar.DEBUG_API:
Registrar.registerMessage('new endpoint %s' % self.next_endpoint)
self.prev_response = self.service.get(self.next_endpoint)
# handle API errors
if self.prev_response.status_code in range(400, 500):
raise ConnectionError('api call failed: %dd with %s' %( self.prev_response.status_code, self.prev_response.text))
# can still 200 and fail
try:
prev_response_json = self.prev_response.json()
except JSONDecodeError:
prev_response_json = {}
e = ConnectionError('api call to %s failed: %s' % (self.next_endpoint, self.prev_response.text))
Registrar.registerError(e)
# if Registrar.DEBUG_API:
# Registrar.registerMessage('first api response: %s' % str(prev_response_json))
if 'errors' in prev_response_json:
raise ConnectionError('first api call returned errors: %s' % (prev_response_json['errors']))
# process API headers
self.processHeaders(self.prev_response)
return prev_response_json
def test_url_get_query_singular(self):
result = UrlUtils.get_query_singular(self.test_url,
'oauth_consumer_key')
self.assertEqual(result, 'ck_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')
result = UrlUtils.get_query_singular(self.test_url, 'filter[limit]')
self.assertEqual(str(result), str(2))
def api_url(self):
components = [self.url, self.api]
return UrlUtils.join_components(components)
def test_url_is_ssl(self):
self.assertTrue(UrlUtils.is_ssl("https://woo.test:8888"))
self.assertFalse(UrlUtils.is_ssl("http://woo.test:8888"))
def request_post_mortem(self, response=None):
"""
Attempt to diagnose what went wrong in a request
"""
reason = None
remedy = None
response_json = {}
try:
response_json = response.json()
except ValueError:
pass
# import pudb; pudb.set_trace()
request_body = {}
request_url = ""
if hasattr(response, 'request'):
if hasattr(response.request, 'url'):
request_url = response.request.url
if hasattr(response.request, 'body'):
request_body = response.request.body
if isinstance(response_json, dict) and ('code' in response_json
or 'message' in response_json):
reason = u" - ".join([
str(response_json.get(key)) for key in ['code', 'message', 'data'] \
if key in response_json
])
if 'code' == 'rest_user_invalid_email':
remedy = "Try checking the email %s doesn't already exist" % \
request_body.get('email')
elif 'code' == 'json_oauth1_consumer_mismatch':
remedy = "Try deleting the cached credentials at %s" % \
self.auth.creds_store
elif 'code' == 'woocommerce_rest_cannot_view':
if not self.auth.query_string_auth:
remedy = "Try enabling query_string_auth"
else:
remedy = (
"This error is super generic and can be caused by just "
"about anything. Here are some things to try: \n"
" - Check that the account which as assigned to your "
"oAuth creds has the correct access level\n"
" - Enable logging and check for error messages in "
"wp-content and wp-content/uploads/wc-logs\n"
" - Check that your query string parameters are valid\n"
" - Make sure your server is not messing with authentication headers\n"
" - Try a different endpoint\n"
" - Try enabling HTTPS and using basic authentication\n"
)
response_headers = {}
if hasattr(response, 'headers'):
response_headers = response.headers
if not reason:
requester_api_url = self.requester.api_url
if hasattr(response, 'links') and response.links:
links = response.links
first_link_key = list(links)[0]
header_api_url = links[first_link_key].get('url', '')
if header_api_url:
header_api_url = StrUtils.eviscerate(header_api_url, '/')
if header_api_url and requester_api_url\
and header_api_url != requester_api_url:
reason = "hostname mismatch. %s != %s" % (
header_api_url, requester_api_url)
header_url = StrUtils.eviscerate(header_api_url, '/')
header_url = StrUtils.eviscerate(header_url,
self.requester.api)
header_url = StrUtils.eviscerate(header_url, '/')
remedy = "try changing url to %s" % header_url
msg = "API call to %s returned \nCODE: %s\nRESPONSE:%s \nHEADERS: %s\nREQ_BODY:%s" % (
request_url, str(
response.status_code), UrlUtils.beautify_response(response),
str(response_headers), str(request_body)[:1000])
if reason:
msg += "\nBecause of %s" % reason
if remedy:
msg += "\n%s" % remedy
raise UserWarning(msg)
def test_url_add_query(self):
self.assertEqual(
"https://woo.test:8888/sdf?param=value&newparam=newvalue",
UrlUtils.add_query("https://woo.test:8888/sdf?param=value", 'newparam', 'newvalue')
)