def processHeaders(self, response): headers = response.headers if Registrar.DEBUG_API: Registrar.registerMessage("headers: %s" % str(headers)) if self.service.namespace == 'wp-api': total_pages_key = 'X-WP-TotalPages' total_items_key = 'X-WP-Total' else: total_pages_key = 'X-WC-TotalPages' total_items_key = 'X-WC-Total' if total_items_key in headers: self.total_pages = int(headers.get(total_pages_key,'')) if total_pages_key in headers: self.total_items = int(headers.get(total_items_key,'')) # if self.progressCounter is None: # self.progressCounter = ProgressCounter(total=self.total_pages) # self.stopNextIteration = True prev_endpoint = self.next_endpoint self.next_endpoint = None for rel, link in response.links.items(): if rel == 'next' and link.get('url'): next_response_url = link['url'] # if Registrar.DEBUG_API: # Registrar.registerMessage('next_response_url: %s' % str(next_response_url)) self.next_page = int(UrlUtils.get_query_singular(next_response_url, 'page')) if not self.next_page: return assert \ self.next_page <= self.total_pages, \ "next page (%s) should be lte total pages (%s)" \ % (str(self.next_page), str(self.total_pages)) self.next_endpoint = UrlUtils.set_query_singular(prev_endpoint,'page', self.next_page) # if Registrar.DEBUG_API: # Registrar.registerMessage('next_endpoint: %s' % str(self.next_endpoint)) if self.next_page: self.offset = (self.limit * self.next_page) + 1
def test_url_get_query_dict_singular(self): result = UrlUtils.get_query_dict_singular(self.test_url) self.assertEquals( result, { 'filter[limit]': '2', 'oauth_nonce': 'c4f2920b0213c43f2e8d3d3333168ec4a22222d1', 'oauth_timestamp': '1481601370', 'oauth_consumer_key': 'ck_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', 'oauth_signature_method': 'HMAC-SHA1', 'oauth_signature': '3ibOjMuhj6JGnI43BQZGniigHh8=', 'page': '2' } )
def test_query_string_endpoint_url(self): query_string_api_params = dict(**self.api_params) query_string_api_params.update(dict(query_string_auth=True)) api = API( **query_string_api_params ) endpoint_url = api.requester.endpoint_url(self.endpoint) endpoint_url = api.auth.get_auth_url(endpoint_url, 'GET') expected_endpoint_url = '%s?consumer_key=%s&consumer_secret=%s' % (self.endpoint, self.consumer_key, self.consumer_secret) expected_endpoint_url = UrlUtils.join_components([self.base_url, self.api_name, self.api_ver, expected_endpoint_url]) self.assertEqual( endpoint_url, expected_endpoint_url ) endpoint_url = api.requester.endpoint_url(self.endpoint) endpoint_url = api.auth.get_auth_url(endpoint_url, 'GET')
def test_APIPutWithSimpleQuery(self): wcapi = API(**self.api_params) response = wcapi.get('products') first_product = (response.json())[0] # from pprint import pformat # print "first product %s" % pformat(response.json()) original_title = first_product['name'] product_id = first_product['id'] nonce = str(random.random()) response = wcapi.put('products/%s?page=2&per_page=5' % (product_id), {"name":str(nonce)}) request_params = UrlUtils.get_query_dict_singular(response.request.url) response_obj = response.json() self.assertEqual(response_obj['name'], str(nonce)) self.assertEqual(request_params['per_page'], '5') wcapi.put('products/%s' % (product_id), {"name":original_title})
def test_APIPutWithSimpleQuery(self): wcapi = API(**self.api_params) response = wcapi.get('products') first_product = (response.json())['products'][0] original_title = first_product['title'] product_id = first_product['id'] nonce = b"%f" % (random.random()) response = wcapi.put('products/%s?filter%%5Blimit%%5D=5' % (product_id), {"product": {"title": text_type(nonce)}}) request_params = UrlUtils.get_query_dict_singular(response.request.url) response_obj = response.json() self.assertEqual(response_obj['product']['title'], text_type(nonce)) self.assertEqual(request_params['filter[limit]'], text_type(5)) wcapi.put('products/%s' % (product_id), {"product": {"title": original_title}})
def test_sorted_params(self): # Example given in oauth.net: oauthnet_example_sorted = [('a', '1'), ('c', 'hi%%20there'), ('f', '25'), ('f', '50'), ('f', 'a'), ('z', 'p'), ('z', 't')] oauthnet_example = copy(oauthnet_example_sorted) random.shuffle(oauthnet_example) # oauthnet_example_sorted = [ # ('a', '1'), # ('c', 'hi%%20there'), # ('f', '25'), # ('z', 'p'), # ] self.assertEqual(UrlUtils.sorted_params(oauthnet_example), oauthnet_example_sorted)
def __init__(self, service, endpoint): assert isinstance(service, WPAPI_Service) self.service = service self.next_endpoint = endpoint self.prev_response = None self.total_pages = None self.total_items = None # self.progressCounter = None endpoint_queries = UrlUtils.get_query_dict_singular(endpoint) # print "endpoint_queries:", endpoint_queries self.next_page = None if 'page' in endpoint_queries: self.next_page = int(endpoint_queries['page']) self.limit = 10 if 'fliter[limit]' in endpoint_queries: self.limit = int(endpoint_queries['filter[limit]']) # print "slave limit set to to ", self.limit self.offset = None if 'filter[offset]' in endpoint_queries: self.offset = int(endpoint_queries['filter[offset]'])
def get_verifier(self, request_token=None, wp_user=None, wp_pass=None): """ pretends to be a browser, uses the authorize auth link, submits user creds to WP login form to get verifier string from access token """ if request_token is None: request_token = self.request_token assert request_token, "need a valid request_token for this step" if wp_user is None and self.wp_user: wp_user = self.wp_user if wp_pass is None and self.wp_pass: wp_pass = self.wp_pass authorize_url = self.authentication['oauth1']['authorize'] authorize_url = UrlUtils.add_query(authorize_url, 'oauth_token', request_token) # we're using a different session from the usual API calls # (I think the headers are incompatible?) # self.requester.get(authorize_url) authorize_session = requests.Session() login_form_response = authorize_session.get(authorize_url) try: login_form_action, login_form_data = self.get_form_info( login_form_response, 'loginform') except AssertionError, e: #try to parse error login_form_soup = BeautifulSoup(login_form_response.text, 'lxml') error = login_form_soup.select_one('div#login_error') if error and "invalid token" in error.string.lower(): raise UserWarning("Invalid token: %s" % repr(request_token)) else: raise UserWarning( "could not parse login form. Site is misbehaving. Original error: %s " \ % str(e) )
def endpoint_url(self, endpoint): endpoint = StrUtils.decapitate(endpoint, '/') return UrlUtils.join_components( [self.url, self.api, self.api_version, endpoint])
def api_url(self): return UrlUtils.join_components([self.url, self.api])
def get_verifier(self, request_token=None, wp_user=None, wp_pass=None): """ pretends to be a browser, uses the authorize auth link, submits user creds to WP login form to get verifier string from access token """ if request_token is None: request_token = self.request_token assert request_token, "need a valid request_token for this step" if wp_user is None and self.wp_user: wp_user = self.wp_user if wp_pass is None and self.wp_pass: wp_pass = self.wp_pass authorize_url = self.authentication['oauth1']['authorize'] authorize_url = UrlUtils.add_query(authorize_url, 'oauth_token', request_token) # we're using a different session from the usual API calls # (I think the headers are incompatible?) # self.requester.get(authorize_url) authorize_session = requests.Session() login_form_response = authorize_session.get(authorize_url) login_form_params = { 'username': wp_user, 'password': wp_pass, 'token': request_token } try: login_form_action, login_form_data = self.get_form_info( login_form_response, 'loginform') except AssertionError as exc: self.parse_login_form_error(login_form_response, exc, **login_form_params) for name, values in login_form_data.items(): if name == 'log': login_form_data[name] = wp_user elif name == 'pwd': login_form_data[name] = wp_pass else: login_form_data[name] = values[0] assert 'log' in login_form_data, 'input for user login did not appear on form' assert 'pwd' in login_form_data, 'input for user password did not appear on form' # print "submitting login form to %s : %s" % (login_form_action, str(login_form_data)) confirmation_response = authorize_session.post(login_form_action, data=login_form_data, allow_redirects=True) try: authorize_form_action, authorize_form_data = self.get_form_info( confirmation_response, 'oauth1_authorize_form') except AssertionError as exc: self.parse_login_form_error(confirmation_response, exc, **login_form_params) for name, values in authorize_form_data.items(): if name == 'wp-submit': assert \ 'authorize' in values, \ "apparently no authorize button, only %s" % str(values) authorize_form_data[name] = 'authorize' else: authorize_form_data[name] = values[0] assert 'wp-submit' in login_form_data, 'authorize button did not appear on form' final_response = authorize_session.post(authorize_form_action, data=authorize_form_data, allow_redirects=False) assert \ final_response.status_code == 302, \ "was not redirected by authorize screen, was %d instead. something went wrong" \ % final_response.status_code assert 'location' in final_response.headers, "redirect did not provide redirect location in header" final_location = final_response.headers['location'] # At this point we can chose to follow the redirect if the user wants, # or just parse the verifier out of the redirect url. # open to suggestions if anyone has any :) final_location_queries = parse_qs(urlparse(final_location).query) assert \ 'oauth_verifier' in final_location_queries, \ "oauth verifier not provided in final redirect: %s" % final_location self._oauth_verifier = final_location_queries['oauth_verifier'][0] return self._oauth_verifier
def get_signature_base_string(cls, method, params, url): base_request_uri = quote(UrlUtils.substitute_query(url), "") query_string = quote(cls.flatten_params(params), '~') return "&".join([method, base_request_uri, query_string])
def test_flatten_params(self): self.assertEqual( UrlUtils.flatten_params(self.twitter_params_raw), self.twitter_param_string )
def test_url_del_query_singular(self): result = UrlUtils.del_query_singular(self.test_url, 'filter[limit]') expected = "http://ich.local:8888/woocommerce/wc-api/v3/products?oauth_consumer_key=ck_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX&oauth_nonce=c4f2920b0213c43f2e8d3d3333168ec4a22222d1&oauth_signature=3ibOjMuhj6JGnI43BQZGniigHh8%3D&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1481601370&page=2" self.assertEqual(result, expected)
def api_ver_url_no_port(self): return UrlUtils.remove_port(self.api_ver_url)
def is_ssl(self): return UrlUtils.is_ssl(self.url)
def test_flatten_params(self): self.assertEqual( StrUtils.to_binary(UrlUtils.flatten_params( self.twitter_params_raw)), StrUtils.to_binary(self.twitter_param_string))
def next(self): if Registrar.DEBUG_API: Registrar.registerMessage('start') if self.next_endpoint is None: if Registrar.DEBUG_API: Registrar.registerMessage('stopping due to no next endpoint') raise StopIteration() # get API response try: self.prev_response = self.service.get(self.next_endpoint) except ReadTimeout as e: # instead of processing this endoint, do the page product by product if self.limit > 1: new_limit = 1 if Registrar.DEBUG_API: Registrar.registerMessage('reducing limit in %s' % self.next_endpoint) self.next_endpoint = UrlUtils.set_query_singular( self.next_endpoint, 'filter[limit]', new_limit ) self.next_endpoint = UrlUtils.del_query_singular( self.next_endpoint, 'page' ) if self.offset: self.next_endpoint = UrlUtils.set_query_singular( self.next_endpoint, 'filter[offset]', self.offset ) self.limit = new_limit # endpoint_queries = parse_qs(urlparse(self.next_endpoint).query) # endpoint_queries = dict([ # (key, value[0]) for key, value in endpoint_queries.items() # ]) # endpoint_queries['filter[limit]'] = 1 # if self.next_page: # endpoint_queries['page'] = 10 * self.next_page # print "endpoint_queries: ", endpoint_queries # self.next_endpoint = UrlUtils.substitute_query( # self.next_endpoint, # urlencode(endpoint_queries) # ) if Registrar.DEBUG_API: Registrar.registerMessage('new endpoint %s' % self.next_endpoint) self.prev_response = self.service.get(self.next_endpoint) # handle API errors if self.prev_response.status_code in range(400, 500): raise ConnectionError('api call failed: %dd with %s' %( self.prev_response.status_code, self.prev_response.text)) # can still 200 and fail try: prev_response_json = self.prev_response.json() except JSONDecodeError: prev_response_json = {} e = ConnectionError('api call to %s failed: %s' % (self.next_endpoint, self.prev_response.text)) Registrar.registerError(e) # if Registrar.DEBUG_API: # Registrar.registerMessage('first api response: %s' % str(prev_response_json)) if 'errors' in prev_response_json: raise ConnectionError('first api call returned errors: %s' % (prev_response_json['errors'])) # process API headers self.processHeaders(self.prev_response) return prev_response_json
def test_url_get_query_singular(self): result = UrlUtils.get_query_singular(self.test_url, 'oauth_consumer_key') self.assertEqual(result, 'ck_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX') result = UrlUtils.get_query_singular(self.test_url, 'filter[limit]') self.assertEqual(str(result), str(2))
def api_url(self): components = [self.url, self.api] return UrlUtils.join_components(components)
def test_url_is_ssl(self): self.assertTrue(UrlUtils.is_ssl("https://woo.test:8888")) self.assertFalse(UrlUtils.is_ssl("http://woo.test:8888"))
def request_post_mortem(self, response=None): """ Attempt to diagnose what went wrong in a request """ reason = None remedy = None response_json = {} try: response_json = response.json() except ValueError: pass # import pudb; pudb.set_trace() request_body = {} request_url = "" if hasattr(response, 'request'): if hasattr(response.request, 'url'): request_url = response.request.url if hasattr(response.request, 'body'): request_body = response.request.body if isinstance(response_json, dict) and ('code' in response_json or 'message' in response_json): reason = u" - ".join([ str(response_json.get(key)) for key in ['code', 'message', 'data'] \ if key in response_json ]) if 'code' == 'rest_user_invalid_email': remedy = "Try checking the email %s doesn't already exist" % \ request_body.get('email') elif 'code' == 'json_oauth1_consumer_mismatch': remedy = "Try deleting the cached credentials at %s" % \ self.auth.creds_store elif 'code' == 'woocommerce_rest_cannot_view': if not self.auth.query_string_auth: remedy = "Try enabling query_string_auth" else: remedy = ( "This error is super generic and can be caused by just " "about anything. Here are some things to try: \n" " - Check that the account which as assigned to your " "oAuth creds has the correct access level\n" " - Enable logging and check for error messages in " "wp-content and wp-content/uploads/wc-logs\n" " - Check that your query string parameters are valid\n" " - Make sure your server is not messing with authentication headers\n" " - Try a different endpoint\n" " - Try enabling HTTPS and using basic authentication\n" ) response_headers = {} if hasattr(response, 'headers'): response_headers = response.headers if not reason: requester_api_url = self.requester.api_url if hasattr(response, 'links') and response.links: links = response.links first_link_key = list(links)[0] header_api_url = links[first_link_key].get('url', '') if header_api_url: header_api_url = StrUtils.eviscerate(header_api_url, '/') if header_api_url and requester_api_url\ and header_api_url != requester_api_url: reason = "hostname mismatch. %s != %s" % ( header_api_url, requester_api_url) header_url = StrUtils.eviscerate(header_api_url, '/') header_url = StrUtils.eviscerate(header_url, self.requester.api) header_url = StrUtils.eviscerate(header_url, '/') remedy = "try changing url to %s" % header_url msg = "API call to %s returned \nCODE: %s\nRESPONSE:%s \nHEADERS: %s\nREQ_BODY:%s" % ( request_url, str( response.status_code), UrlUtils.beautify_response(response), str(response_headers), str(request_body)[:1000]) if reason: msg += "\nBecause of %s" % reason if remedy: msg += "\n%s" % remedy raise UserWarning(msg)
def test_url_add_query(self): self.assertEqual( "https://woo.test:8888/sdf?param=value&newparam=newvalue", UrlUtils.add_query("https://woo.test:8888/sdf?param=value", 'newparam', 'newvalue') )