def _split_url(url): "Splits given url to url base and params converted to list of tuples" split = parse.urlsplit(url) base = parse.urlunsplit((split.scheme, split.netloc, split.path, 0, 0)) params = parse.parse_qsl(split.query, True) return base, params
def _x_user_parser(user, data): _user = data.get('response', {}).get('user', {}) user.username = user.id = _user.get('name') user.link = _user.get('blogs', [{}])[0].get('url') if user.link: _host = parse.urlsplit(user.link).netloc user.picture = 'http://api.tumblr.com/v2/blog/{0}/avatar/512'.format(_host) return user
def _x_user_parser(user, data): _user = data.get('response', {}).get('user', {}) user.username = user.id = _user.get('name') user.link = _user.get('blogs', [{}])[0].get('url') if user.link: _host = parse.urlsplit(user.link).netloc user.picture = 'http://api.tumblr.com/v2/blog/{0}/avatar/512'.format( _host) return user
def _fetch(self, url, method='GET', params=None, headers=None, body='', max_redirects=5, content_parser=None): """ Fetches a URL. :param str url: The URL to fetch. :param str method: HTTP method of the request. :param dict params: Dictionary of request parameters. :param dict headers: HTTP headers of the request. :param str body: Body of ``POST``, ``PUT`` and ``PATCH`` requests. :param int max_redirects: Number of maximum HTTP redirects to follow. :param function content_parser: A callable to be used to parse the :attr:`.Response.data` from :attr:`.Response.content`. """ params = params or {} params.update(self.access_params) headers = headers or {} headers.update(self.access_headers) scheme, host, path, query, fragment = parse.urlsplit(url) query = parse.urlencode(params) if method in ('POST', 'PUT', 'PATCH'): if not body: # Put querystring to body body = query query = '' headers.update({'Content-Type': 'application/x-www-form-urlencoded'}) print(path, query) request_path = parse.urlunsplit(('', '', path or '', query or '', '')) self._log(logging.DEBUG, ' \u251C\u2500 host: {0}'.format(host)) self._log(logging.DEBUG, ' \u251C\u2500 path: {0}'.format(request_path)) self._log(logging.DEBUG, ' \u251C\u2500 method: {0}'.format(method)) self._log(logging.DEBUG, ' \u251C\u2500 body: {0}'.format(body)) self._log(logging.DEBUG, ' \u251C\u2500 params: {0}'.format(params)) self._log(logging.DEBUG, ' \u2514\u2500 headers: {0}'.format(headers)) # Connect if scheme.lower() == 'https': connection = http_client.HTTPSConnection(host) else: connection = http_client.HTTPConnection(host) try: connection.request(method, request_path, body, headers) except Exception as e: raise FetchError('Could not connect!', original_message=e.message, url=request_path) response = connection.getresponse() location = response.getheader('Location') if response.status in (300, 301, 302, 303, 307) and location: if location == url: raise FetchError('Url redirects to itself!', url=location, status=response.status) elif max_redirects > 0: remaining_redirects = max_redirects - 1 self._log(logging.DEBUG, 'Redirecting to {0}'.format(url)) self._log(logging.DEBUG, 'Remaining redirects: {0}' .format(remaining_redirects)) # Call this method again. response = self._fetch(url=location, params=params, method=method, headers=headers, max_redirects=remaining_redirects) else: raise FetchError('Max redirects reached!', url=location, status=response.status) else: self._log(logging.DEBUG, 'Got response:') self._log(logging.DEBUG, ' \u251C\u2500 url: {0}'.format(url)) self._log(logging.DEBUG, ' \u251C\u2500 status: {0}'.format(response.status)) self._log(logging.DEBUG, ' \u2514\u2500 headers: {0}'.format(response.getheaders())) return authomatic.core.Response(response, content_parser)
def _fetch(self, url, method='GET', params=None, headers=None, body='', max_redirects=5, content_parser=None): """ Fetches a URL. :param str url: The URL to fetch. :param str method: HTTP method of the request. :param dict params: Dictionary of request parameters. :param dict headers: HTTP headers of the request. :param str body: Body of ``POST``, ``PUT`` and ``PATCH`` requests. :param int max_redirects: Number of maximum HTTP redirects to follow. :param function content_parser: A callable to be used to parse the :attr:`.Response.data` from :attr:`.Response.content`. """ params = params or {} params.update(self.access_params) headers = headers or {} headers.update(self.access_headers) scheme, host, path, query, fragment = parse.urlsplit(url) query = parse.urlencode(params) if method in ('POST', 'PUT', 'PATCH'): if not body: # Put querystring to body body = query query = '' headers.update( {'Content-Type': 'application/x-www-form-urlencoded'}) request_path = parse.urlunsplit(('', '', path or '', query or '', '')) self._log(logging.DEBUG, ' \u251C\u2500 host: {0}'.format(host)) self._log(logging.DEBUG, ' \u251C\u2500 path: {0}'.format(request_path)) self._log(logging.DEBUG, ' \u251C\u2500 method: {0}'.format(method)) self._log(logging.DEBUG, ' \u251C\u2500 body: {0}'.format(body)) self._log(logging.DEBUG, ' \u251C\u2500 params: {0}'.format(params)) self._log(logging.DEBUG, ' \u2514\u2500 headers: {0}'.format(headers)) # Connect if scheme.lower() == 'https': connection = http_client.HTTPSConnection(host) else: connection = http_client.HTTPConnection(host) try: connection.request(method, request_path, body, headers) except Exception as e: raise FetchError('Could not connect!', original_message=e.message, url=request_path) response = connection.getresponse() location = response.getheader('Location') if response.status in (300, 301, 302, 303, 307) and location: if location == url: raise FetchError('Url redirects to itself!', url=location, status=response.status) elif max_redirects > 0: remaining_redirects = max_redirects - 1 self._log(logging.DEBUG, 'Redirecting to {0}'.format(url)) self._log( logging.DEBUG, 'Remaining redirects: {0}'.format(remaining_redirects)) # Call this method again. response = self._fetch(url=location, params=params, method=method, headers=headers, max_redirects=remaining_redirects) else: raise FetchError('Max redirects reached!', url=location, status=response.status) else: self._log(logging.DEBUG, 'Got response:') self._log(logging.DEBUG, ' \u251C\u2500 url: {0}'.format(url)) self._log(logging.DEBUG, ' \u251C\u2500 status: {0}'.format(response.status)) self._log( logging.DEBUG, ' \u2514\u2500 headers: {0}'.format(response.getheaders())) return authomatic.core.Response(response, content_parser)