def __init__(self, host, endpoint=None, encoding='utf-8', persist_cookies=None, connections=1): ''' Args: host (str): The top level domain to which all of the requests will be made. Example: 'https://example.org' endpoint (str): The base uri can be augmented further. Example: '/chat'. Calling one of the http method methods without a further path, like .get(), would result in a request to 'https://example.org/chat' encoding (str): The encoding asks'll try to use on response bodies. persist_cookies (bool): Passing True turns on browserishlike stateful cookie behaviour, returning cookies to the host when appropriate. connections (int): The max number of concurrent connections to the host asks will allow its self to have. The default number of connections is ONE. You *WILL* want to change this value to suit your application and the limits of the remote host you're working with. ''' self.encoding = encoding self.endpoint = endpoint self.host = host self.port = None if persist_cookies is True: self.cookie_tracker_obj = CookieTracker() else: self.cookie_tracker_obj = persist_cookies self.conn_pool = SocketQ(maxlen=connections) self.checked_out_sockets = SocketQ(maxlen=connections) self.sema = curio.BoundedSemaphore(value=connections) self.in_connection_counter = 0
def __init__(self, encoding='utf-8', persist_cookies=None, connections=20): ''' Args: encoding (str): The encoding asks'll try to use on response bodies. persist_cookies (bool): Passing True turns on browserishlike stateful cookie behaviour, returning cookies to the host when appropriate. connections (int): The max number of concurrent connections to the host asks will allow its self to have. The default number of connections is 20. You may increase or decrease this value as you see fit. ''' self.encoding = encoding if persist_cookies is True: self.cookie_tracker_obj = CookieTracker() else: self.cookie_tracker_obj = persist_cookies self.conn_pool = SocketQ(maxlen=connections) self.checked_out_sockets = SocketQ(maxlen=connections) self.sema = curio.BoundedSemaphore(value=1) self.in_connection_counter = 0
def __init__(self): self._lock = curio.BoundedSemaphore()
import curio import curio_http import json import requests from typing import List, Dict from settings import Settings MAX_CONNECTIONS_PER_HOST = 6 sema = curio.BoundedSemaphore(MAX_CONNECTIONS_PER_HOST) class EventCodeFetcher: def __init__(self): self.settings = Settings() def list_of_all_event_codes(self): url = 'https://www.thebluealliance.com/api/v2/events/2017' headers = {'X-TBA-App-Id': self.settings.app_id} events = requests.get(url, headers=headers).json() return [event['key'] for event in events] class DataFetcher: def __init__(self): self.settings = Settings() async def fetch_one(self, url): headers = {'X-TBA-App-Id': self.settings.app_id} #async with curio_http.ClientSession() as session: async with sema, curio_http.ClientSession() as session: response = await session.get(url, headers=headers)