Example #1
0
    def request(self, method, url, params=None, headers=None, **aio_kwargs):
        """ Request OAuth resouce. """
        oparams = {
            'oauth_consumer_key': self.consumer_key,
            'oauth_nonce': sha1(str(random()).encode('ascii')).hexdigest(),
            'oauth_signature_method': self.signature.name,
            'oauth_timestamp': int(time.time()),
            'oauth_version': self.version,
        }
        oparams.update(params or {})

        if self.oauth_token:
            oparams['oauth_token'] = self.oauth_token

        url = self._get_url(url)

        oparams['oauth_signature'] = self.signature.sign(
            self.consumer_secret,
            method,
            url,
            oauth_token_secret=self.oauth_token_secret,
            **oparams)
        self.logger.debug("%s %s" % (url, oparams))
        return aiorequest(method,
                          url,
                          params=oparams,
                          headers=headers,
                          **aio_kwargs)
Example #2
0
    def request(self,
                method,
                url,
                params=None,
                headers=None,
                timeout=10,
                loop=None,
                **aio_kwargs):
        """Request OAuth2 resource."""
        url = self._get_url(url)
        params = params or {}

        if self.access_token:
            params[self.access_token_key] = self.access_token

        headers = headers or {
            'Accept': 'application/json',
            'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
        }
        return asyncio.wait_for(aiorequest(method,
                                           url,
                                           params=params,
                                           headers=headers,
                                           loop=loop,
                                           **aio_kwargs),
                                timeout,
                                loop=loop)
Example #3
0
 def request(self,
             method,
             url,
             params=None,
             headers=None,
             timeout=10,
             loop=None,
             **aio_kwargs):
     """Request OAuth2 resource."""
     url = self._get_url(url)
     if self.access_token:
         headers = headers or {'Accept': 'application/json'}
         headers['Authorization'] = "Bearer {}".format(self.access_token)
         auth = None
     else:
         auth = BasicAuth(self.client_id, self.client_secret)
         headers = headers or {
             'Accept': 'application/json',
             'Content-Type':
             'application/x-www-form-urlencoded;charset=UTF-8',
         }
     # noinspection PyArgumentList
     return asyncio.wait_for(aiorequest(method,
                                        url,
                                        params=params,
                                        headers=headers,
                                        auth=auth,
                                        loop=loop,
                                        **aio_kwargs),
                             timeout,
                             loop=loop)
Example #4
0
    def request(self, method, url, params=None, headers=None, **aio_kwargs):
        """ Request OAuth2 resource. """
        url = self._get_url(url)
        params = params or {}

        if self.access_token:
            params[self.access_token_key] = self.access_token

        headers = headers or {
            'Accept': 'application/json',
            'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
        }
        return aiorequest(method, url, params=params, headers=headers, **aio_kwargs)
Example #5
0
 def request(self, method, url, params=None, headers=None, timeout=10, **aio_kwargs):
     """Request OAuth2 resource."""
     url = self._get_url(url)
     if self.access_token:
         headers = headers or {
                         'Accept': 'application/json',
                     }
         headers['Authorization'] = "Bearer {}".format(self.access_token)
         auth = None
     else:
         auth = BasicAuth(self.client_id, self.client_secret)
         headers = headers or {
             'Accept': 'application/json',
             'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
         }
     # noinspection PyArgumentList
     return asyncio.wait_for(
         aiorequest(method, url, params=params, headers=headers, auth=auth, **aio_kwargs), timeout)
Example #6
0
    def request(self,
                method,
                url,
                params=None,
                headers=None,
                timeout=10,
                loop=None,
                **aio_kwargs):
        """Make a request to provider."""
        oparams = {
            'oauth_consumer_key': self.consumer_key,
            'oauth_nonce': sha1(str(random()).encode('ascii')).hexdigest(),
            'oauth_signature_method': self.signature.name,
            'oauth_timestamp': str(int(time.time())),
            'oauth_version': self.version,
        }
        oparams.update(params or {})

        if self.oauth_token:
            oparams['oauth_token'] = self.oauth_token

        url = self._get_url(url)

        if urlsplit(url).query:
            raise ValueError(
                'Request parameters should be in the "params" parameter, not inlined in the URL'
            )

        oparams['oauth_signature'] = self.signature.sign(
            self.consumer_secret,
            method,
            url,
            oauth_token_secret=self.oauth_token_secret,
            **oparams)
        self.logger.debug("%s %s", url, oparams)
        return asyncio.wait_for(aiorequest(method,
                                           url,
                                           params=oparams,
                                           headers=headers,
                                           loop=loop,
                                           **aio_kwargs),
                                timeout,
                                loop=loop)
Example #7
0
    async def execute(request: RetrievalRequest) -> None:
        nonlocal requests_executed
        url = request.url
        filePath = request.path
        os.makedirs(os.path.dirname(filePath), exist_ok=True)
        logging.debug(f"Requesting {filePath} from {url}")
        try:
            async with requests_executed_lock:
                remaining_count = len(requests_remaining) - requests_executed

            if (remaining_count <= 10
                    or math.ceil(len(retrieval_requests) / 10) %
                    remaining_count == 0):
                logging.info(f"{remaining_count} remaining")

            async with aiorequest("get",
                                  url,
                                  headers={
                                      "User-Agent": get_random_user_agent()
                                  }) as response:
                is_expected_type = is_expected_response_type(
                    response, request.expected_types)
                if is_expected_type is None:
                    logging.info(
                        f"Cannot determine response type, may not be the desired response for {url}"
                    )
                else:
                    if is_expected_type:
                        logging.debug(
                            f"Response is of expected type for {url}")
                        out = open(filePath, "wb")
                        out.write(await response.read())
                        out.close()
                    else:
                        raise ValueError(
                            f"Response for {url} is not the expected type")
        except Exception as ex:
            logging.debug(f"Error fetching {url}: {ex}")
            requests_failed.append(request)
        finally:
            async with requests_executed_lock:
                requests_executed += 1
Example #8
0
    def request(self, method, url, params=None, headers=None, **aio_kwargs):
        """ Request OAuth resouce. """
        oparams = {
            'oauth_consumer_key': self.consumer_key,
            'oauth_nonce': sha1(str(random()).encode('ascii')).hexdigest(),
            'oauth_signature_method': self.signature.name,
            'oauth_timestamp': int(time.time()),
            'oauth_version': self.version,
        }
        oparams.update(params or {})

        if self.oauth_token:
            oparams['oauth_token'] = self.oauth_token

        url = self._get_url(url)

        oparams['oauth_signature'] = self.signature.sign(
            self.consumer_secret, method, url,
            oauth_token_secret=self.oauth_token_secret, **oparams)
        self.logger.debug("%s %s" % (url, oparams))
        return aiorequest(method, url, params=oparams, headers=headers, **aio_kwargs)
Example #9
0
 def _do_request(self, method, url, **aio_kwargs):
     print(method, url)
     return aiorequest(method, url, **aio_kwargs)