def __init__(self, item_list: ItemList):
self.queue = asyncio.Queue()
self.event_loop = asyncio.get_event_loop()
self.client_session = aiohttp.ClientSession()
self.item_list = item_list
self.backends = [
BackendPoolWorker(PoeTrade(item_list), self.event_loop, Throttler(10, 1)),
BackendPoolWorker(PoeOfficial(item_list), self.event_loop, Throttler(2, 3)),
]
async def token_transfers_generator(
self,
address: str = None,
contract_address: str = None,
throttler: Throttler = None,
block_limit: int = 50,
offset: int = 3,
start_block: int = 0,
end_block: int = None,
) -> AsyncIterator[Dict]:
if end_block is None:
end_block = int(await self._client.proxy.block_number(), 16)
if throttler is None:
throttler = Throttler(rate_limit=5, period=1.0)
for sblock, eblock in self._generate_intervals(start_block, end_block,
block_limit):
async for transfer in self._parse_by_pages(
address=address,
contract_address=contract_address,
start_block=sblock,
end_block=eblock,
offset=offset,
throttler=throttler):
yield transfer
async def crawl(self):
"""Run the crawler until all work is done.
This is the main execution coroutine for the crawler.
"""
# Queues must be created inside event loop (i.e. don't place in __init__)
self.request_queue = asyncio.Queue()
# Clientsession should be created from async function (i.e. don't place in __init__)
self.session = aiohttp.ClientSession()
self.throttler = Throttler(rate_limit=10, period=1)
for url in self.urls:
await self.schedule(Job(url, self.stage1_request_category_data))
self.worker_tasks = [asyncio.create_task(self.worker(name=i)) for i in range(self.max_workers)]
# Only needed if Windows and Python version < 3.8
self.health_tasks = [asyncio.create_task(self.heartbeat()), ]
# When all work is done, exit.
await self.request_queue.join()
for worker in self.worker_tasks:
worker.cancel()
await self.close()
async def _run():
async with TelegramClient('name', api_id, api_hash) as client:
queue = Queue()
throttler = Throttler(rate_limit=5, period=10)
# asyncio.create_task(alert_mess_worker(queue, throttler, client,des=destination_channel))
asyncio.create_task(slack_alert_worker(queue, throttler))
await client.send_message(
destination_channel,
'INIT - %s - %s' % (k2_group_id, '+'.join(vips)))
@client.on(events.NewMessage(chats=k2_group_id, from_users=vips))
async def handler(event):
queue.put_nowait(event)
# alert important
hour_tz = datetime.utcnow().hour + 9
hour_tz = hour_tz - divmod(hour_tz, 24)[0] * 24
if hour_tz >= 0 and hour_tz <= 9 and type(
event.message.media) in [
MessageMediaWebPage, MessageMediaPhoto
]:
requests.get("https://vybit.net/trigger/vpgobhuhwbg4hk7u")
await client.run_until_disconnected()
# * alert if disconnected
slack_client.chat_postMessage(channel="C01STR8P9ST",
text=('DISCONNECTED !!!'))
def __init__(self):
self.work_futures = dict()
self.next_queue = 1
self.service_throttlers = defaultdict(lambda: Throttler(
rate_limit=BpowServer.MAX_SERVICE_REQUESTS_PER_SECOND * 10,
period=10))
self.database = BpowRedis(
f"redis://{os.getenv('REDIS_HOST', 'localhost')}", loop)
self.mqtt = BpowMQTT(config.mqtt_uri,
loop,
self.client_handler,
self.database,
logger=logger)
if config.use_websocket:
self.websocket = WebsocketClient(config.websocket_uri,
self.block_arrival_ws_handler,
logger=logger)
else:
self.websocket = None
if config.use_nano_websocket:
self.nano_ws = WebsocketClient(config.nano_websocket_uri,
self.block_arrival_ws_handler_nano,
logger=logger)
else:
self.nano_ws = None
async def crawl_skus():
skus = read_json(data_path / 'skus.json')
gpus = read_json(data_path / 'gpus.json')
tasks = []
throttler = Throttler(rate_limit=10)
pbar = tqdm(total=len(skus) * len(gpus), desc="Crawling SKUs", unit="SKU")
async with aiohttp.ClientSession(headers=headers) as client:
for promo_locale in skus.keys():
for gpu_name, gpu_data in gpus.items():
tasks.append(retrieve_sku(
client, promo_locale,
gpu_name, gpu_data,
throttler, pbar))
results = await asyncio.gather(*tasks, return_exceptions=True)
for locale, model, new_sku in results:
current_sku = skus[locale][model]
if new_sku is not None and current_sku != new_sku:
pbar.write(
f'Updating SKU for {model} in {locale}: {current_sku} -> {new_sku}')
skus[locale][model] = new_sku
elif new_sku is not None and current_sku == new_sku:
pbar.write(
f'SKU confirmed for {model} in {locale}: {new_sku}')
with open(data_path / 'skus.json', 'w+') as f:
f.write(json.dumps(skus, indent=4))
async def test_rate_limiting(self, rate_limit, workers_to_spawn,
event_loop):
throttler = Throttler(rate_limit)
logs = deque()
tasks = [
event_loop.create_task(self.worker(throttler, logs))
for _ in range(workers_to_spawn)
]
started = time.time()
while True:
now = time.time()
if now - started >= 5.0:
break
while logs:
if now - logs[0] > 1.0:
logs.popleft()
else:
break
assert len(logs) <= rate_limit
await asyncio.sleep(0.05)
for task in tasks:
task.cancel()
async def endpoint_execute(self, endpoints):
"""Asynchronously calls each endpoint and returns the JSON responses
Args:
endpoints (list): List of endpoints to get
Returns:
The list of JSON responses corresponding to each endpoint
"""
throttler = Throttler(rate_limit=self.rate_limit, period=self.rate_period)
ssl_ctx = ssl.create_default_context(cafile=certifi.where())
connector = aiohttp.TCPConnector(limit_per_host=self.connection_limit, ssl=ssl_ctx)
session = aiohttp.ClientSession(connector=connector)
# Asnycio create tasks for each endpoint
try:
sem = asyncio.Semaphore(self.connection_limit)
tasks = [asyncio.create_task(self.bound_fetch(sem, endpoint, session, throttler)) for endpoint in endpoints]
[await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(endpoints))]
ret = [t.result() for t in tasks]
finally:
await session.close()
return ret
def __init__(self, bot: IBot, name: str):
super().__init__(name)
self.bot = bot
self.disconnected = False
self.throttle = Throttler(rate_limit=100, period=1)
self.sasl_state = SASLResult.NONE
self.last_read = monotonic()
self._sent_count: int = 0
self._send_queue: PriorityQueue[SentLine] = PriorityQueue()
self.desired_caps: Set[ICapability] = set([])
self._read_queue: Deque[Line] = deque()
self._process_queue: Deque[Tuple[Line, Optional[Emit]]] = deque()
self._ping_sent = False
self._read_lguard = RLock()
self.read_lock = self._read_lguard
self._read_lwork = asyncio.Lock()
self._wait_for = asyncio.Event()
self._pending_who: Deque[str] = deque()
self._alt_nicks: List[str] = []
def __init__(
self,
host: str,
app_key: str,
websession: aiohttp.ClientSession | None = None,
) -> None:
"""
Initialize the Bridge instance.
Parameters:
`host`: the hostname or IP-address of the bridge as string.
`app_key`: provide the hue appkey/username for authentication.
`websession`: optionally provide a aiohttp ClientSession.
"""
self._host = host
self._app_key = app_key
self._websession = websession
self._websession_provided = websession is not None
self.logger = logging.getLogger(f"{__package__}[{host}]")
# all api controllers
self._config = None
self._devices = None
self._lights = None
self._scenes = None
self._groups = None
self._sensors = None
# Setup the Throttler/rate limiter for requests to the bridge.
self._throttler = Throttler(rate_limit=THROTTLE_CONCURRENT_REQUESTS,
period=THROTTLE_TIMESPAN)
def __init__(self, service):
self.rsaExecutor = RSAExecutor()
self.throttler = Throttler(
rate_limit=10,
period=5.0,
retry_interval=2.0
)
self.service = service
async def main():
"""Main program for setting up task and throttler
"""
throttler = Throttler(rate_limit=3, period=3)
async with ClientSession() as session:
urls = build_urls(2020)
tasks = [loop.create_task(worker(throttler, session, urls))]
await asyncio.wait(tasks)
def __init__(self, config: Dict):
'''config should be a Python dictionary containing the API key for the Alma Users API as well as the endpoint for looking up a user by Primary ID. '''
self.logger = logging.getLogger('lcpp.alma_requests')
check_config(config=config,
top_level_key='Alma',
config_keys=['apikeys', 'users_endpt'],
obj=self)
# Initialize throttler for Alma's rate limit
self.throttler = Throttler(rate_limit=25)
async def main():
tasks = []
throttler = Throttler(rate_limit=20, period=1)
conn = aiohttp.TCPConnector(limit=30)
async with aiohttp.ClientSession(connector=conn) as session:
for _ in range(100):
tasks.append(fetch(session, URL, throttler, _))
await asyncio.gather(*tasks)
def __init__(self):
self.last_block = None
self.work_futures = dict()
self.service_throttlers = defaultdict(lambda: Throttler(rate_limit=config.throttle*1, period=1))
self.database = DpowRedis("redis://localhost", loop)
self.mqtt = DpowMQTT(config.mqtt_uri, loop, self.client_handler, logger=logger)
if config.websocket_uri:
self.websocket = WebsocketClient(config.websocket_uri, self.block_arrival_ws_handler, logger=logger)
else:
self.websocket = None
self.base_difficulty = config.difficulty or nanolib.work.WORK_DIFFICULTY
async def _fetch_all(self, urls: List[str], rate: Optional[int] = None) -> List[JsonType]:
connector = aiohttp.TCPConnector(verify_ssl=False, limit=100)
throttler = None
if rate:
throttler = Throttler(rate_limit=rate, period=1)
async with aiohttp.ClientSession(connector=connector) as session:
tasks = [self._throttler(session, url, throttler, i) for i, url in enumerate(urls)]
responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))]
responses = sorted(responses, key=lambda x: x[1])
responses = list(map(lambda x: x[0], responses))
return responses
async def _run():
async with TelegramClient('name', api_id, api_hash) as client:
queue = Queue()
throttler = Throttler(rate_limit=5, period=10)
asyncio.create_task(slack_alert_worker(queue, throttler))
@client.on(events.NewMessage(chats=channels))
async def handler(event):
queue.put_nowait(event)
# alert important
await client.run_until_disconnected()
async def on_start(self) -> bool:
"""Handle initialization of the provider based on config."""
# pylint: disable=attribute-defined-outside-init
config = self.mass.config.get_provider_config(self.id)
if not config[CONF_USERNAME] or not config[CONF_PASSWORD]:
LOGGER.debug(
"Username and password not set. Abort load of provider.")
return False
self._username = config[CONF_USERNAME]
self._password = config[CONF_PASSWORD]
self._throttler = Throttler(rate_limit=1, period=1)
return True
async def fetch_offers_async(league, currency_pairs, limit=3):
throttler = Throttler(10)
async with aiohttp.ClientSession() as sess:
tasks = [
asyncio.ensure_future(
fetch_offers_for_pair(sess, throttler, league, p[0], p[1],
limit)) for p in currency_pairs
]
(done, _not_done) = await asyncio.wait(tasks)
results = [task.result() for task in done]
return results
async def _run():
async with TelegramClient('name', api_id, api_hash) as client:
queue = Queue()
throttler = Throttler(rate_limit=5, period=10)
asyncio.create_task(slack_alert_worker(queue, throttler))
@client.on(events.NewMessage(chats=group_id, from_users=vips))
async def handler(event):
queue.put_nowait(event)
await client.run_until_disconnected()
# * alert if disconnected
slack_client.chat_postMessage(
channel=channel,
text=('DISCONNECTED !!!')
)
def __init__(self, config):
self._apps: Deque[Dict[str, Any]] = deque()
self._base_url = 'https://api.intra.42.fr/v2/'
self._auth_url = 'https://api.intra.42.fr/oauth/token'
self._config = config
self._refresher = None
self._logger = logging.getLogger('IntraAPI')
ssl_context = ssl.create_default_context(cafile=certifi.where())
connector = TCPConnector(ssl=ssl_context)
timeout: ClientTimeout = ClientTimeout(total=60)
self.session: ClientSession = ClientSession(connector=connector,
json_serialize=ujson.dumps,
timeout=timeout)
self._throttler = Throttler(rate_limit=20)
def __init__(self):
self.work_futures = dict()
self.service_throttlers = defaultdict(lambda: Throttler(
rate_limit=BpowServer.MAX_SERVICE_REQUESTS_PER_SECOND * 10,
period=10))
self.database = BpowRedis("redis://localhost", loop)
self.mqtt = BpowMQTT(config.mqtt_uri,
loop,
self.client_handler,
logger=logger)
if config.use_websocket:
self.websocket = WebsocketClient(config.websocket_uri,
self.block_arrival_ws_handler,
logger=logger)
else:
self.websocket = None
def run(provider,
# AWS
profile=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
# Azure
user_account=False, service_account=None,
cli=False, msi=False, service_principal=False, file_auth=None,
tenant_id=None, subscription_id=None,
client_id=None, client_secret=None,
username=None, password=None,
# GCP
project_id=None, folder_id=None, organization_id=None, all_projects=False,
# Aliyun
access_key_id=None, access_key_secret=None,
# General
report_name=None, report_dir=None,
timestamp=False,
services=[], skipped_services=[],
result_format='json',
database_name=None, host_ip='127.0.0.1', host_port=8000,
max_workers=10,
regions=[],
excluded_regions=[],
fetch_local=False, update=False,
max_rate=None,
ip_ranges=[], ip_ranges_name_key='name',
ruleset='default.json', exceptions=None,
force_write=False,
debug=False,
quiet=False,
log_file=None,
no_browser=False,
programmatic_execution=True):
"""
Run a scout job in an async event loop.
"""
loop = asyncio.get_event_loop()
# Set the throttler within the loop so it's accessible later on
loop.throttler = Throttler(rate_limit=max_rate if max_rate else 999999, period=1)
loop.set_default_executor(ThreadPoolExecutor(max_workers=max_workers))
result = loop.run_until_complete(_run(**locals())) # pass through all the parameters
loop.close()
return result
def download_throttler(self):
"""
Return the Throttler which can be used to rate limit downloaders.
Upon first access, the Throttler is instantiated and saved internally.
Plugin writers are expected to override when additional configuration of the
DownloaderFactory is needed.
Returns:
Throttler: The instantiated Throttler to be used by get_downloader()
"""
try:
return self._download_throttler
except AttributeError:
if self.rate_limit:
self._download_throttler = Throttler(rate_limit=self.rate_limit)
return self._download_throttler
async def check_worldwide_status():
locales = read_json(data_path / 'locales.json')
dr_locales = set(locale['DRlocale'].replace('_', '-')
for locale in locales.values())
pbar = tqdm(total=len(dr_locales),
desc="Checking worldwide product status", unit="locale")
throttler = Throttler(rate_limit=10)
async with aiohttp.ClientSession(headers=headers) as client:
tasks = []
for dr_locale in dr_locales:
tasks.append(check_product_status(
client, dr_locale, throttler, pbar))
status_results = {res[0]: res[1] for res in await asyncio.gather(*tasks, return_exceptions=True) if res is not None}
with open(data_path / 'status.json', 'w+') as f:
pbar.write(f'Writing status results to data/status.json')
f.write(json.dumps(status_results, indent=4, sort_keys=True))
