def request(self, method, url, params=None, headers=None, **aio_kwargs): """ Request OAuth resouce. """ oparams = { 'oauth_consumer_key': self.consumer_key, 'oauth_nonce': sha1(str(random()).encode('ascii')).hexdigest(), 'oauth_signature_method': self.signature.name, 'oauth_timestamp': int(time.time()), 'oauth_version': self.version, } oparams.update(params or {}) if self.oauth_token: oparams['oauth_token'] = self.oauth_token url = self._get_url(url) oparams['oauth_signature'] = self.signature.sign( self.consumer_secret, method, url, oauth_token_secret=self.oauth_token_secret, **oparams) self.logger.debug("%s %s" % (url, oparams)) return aiorequest(method, url, params=oparams, headers=headers, **aio_kwargs)
def request(self, method, url, params=None, headers=None, timeout=10, loop=None, **aio_kwargs): """Request OAuth2 resource.""" url = self._get_url(url) params = params or {} if self.access_token: params[self.access_token_key] = self.access_token headers = headers or { 'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', } return asyncio.wait_for(aiorequest(method, url, params=params, headers=headers, loop=loop, **aio_kwargs), timeout, loop=loop)
def request(self, method, url, params=None, headers=None, timeout=10, loop=None, **aio_kwargs): """Request OAuth2 resource.""" url = self._get_url(url) if self.access_token: headers = headers or {'Accept': 'application/json'} headers['Authorization'] = "Bearer {}".format(self.access_token) auth = None else: auth = BasicAuth(self.client_id, self.client_secret) headers = headers or { 'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', } # noinspection PyArgumentList return asyncio.wait_for(aiorequest(method, url, params=params, headers=headers, auth=auth, loop=loop, **aio_kwargs), timeout, loop=loop)
def request(self, method, url, params=None, headers=None, **aio_kwargs): """ Request OAuth2 resource. """ url = self._get_url(url) params = params or {} if self.access_token: params[self.access_token_key] = self.access_token headers = headers or { 'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', } return aiorequest(method, url, params=params, headers=headers, **aio_kwargs)
def request(self, method, url, params=None, headers=None, timeout=10, **aio_kwargs): """Request OAuth2 resource.""" url = self._get_url(url) if self.access_token: headers = headers or { 'Accept': 'application/json', } headers['Authorization'] = "Bearer {}".format(self.access_token) auth = None else: auth = BasicAuth(self.client_id, self.client_secret) headers = headers or { 'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', } # noinspection PyArgumentList return asyncio.wait_for( aiorequest(method, url, params=params, headers=headers, auth=auth, **aio_kwargs), timeout)
def request(self, method, url, params=None, headers=None, timeout=10, loop=None, **aio_kwargs): """Make a request to provider.""" oparams = { 'oauth_consumer_key': self.consumer_key, 'oauth_nonce': sha1(str(random()).encode('ascii')).hexdigest(), 'oauth_signature_method': self.signature.name, 'oauth_timestamp': str(int(time.time())), 'oauth_version': self.version, } oparams.update(params or {}) if self.oauth_token: oparams['oauth_token'] = self.oauth_token url = self._get_url(url) if urlsplit(url).query: raise ValueError( 'Request parameters should be in the "params" parameter, not inlined in the URL' ) oparams['oauth_signature'] = self.signature.sign( self.consumer_secret, method, url, oauth_token_secret=self.oauth_token_secret, **oparams) self.logger.debug("%s %s", url, oparams) return asyncio.wait_for(aiorequest(method, url, params=oparams, headers=headers, loop=loop, **aio_kwargs), timeout, loop=loop)
async def execute(request: RetrievalRequest) -> None: nonlocal requests_executed url = request.url filePath = request.path os.makedirs(os.path.dirname(filePath), exist_ok=True) logging.debug(f"Requesting {filePath} from {url}") try: async with requests_executed_lock: remaining_count = len(requests_remaining) - requests_executed if (remaining_count <= 10 or math.ceil(len(retrieval_requests) / 10) % remaining_count == 0): logging.info(f"{remaining_count} remaining") async with aiorequest("get", url, headers={ "User-Agent": get_random_user_agent() }) as response: is_expected_type = is_expected_response_type( response, request.expected_types) if is_expected_type is None: logging.info( f"Cannot determine response type, may not be the desired response for {url}" ) else: if is_expected_type: logging.debug( f"Response is of expected type for {url}") out = open(filePath, "wb") out.write(await response.read()) out.close() else: raise ValueError( f"Response for {url} is not the expected type") except Exception as ex: logging.debug(f"Error fetching {url}: {ex}") requests_failed.append(request) finally: async with requests_executed_lock: requests_executed += 1
def _do_request(self, method, url, **aio_kwargs): print(method, url) return aiorequest(method, url, **aio_kwargs)