from __future__ import absolute_import
import re
import sys
import urllib
from datetime import date, timedelta
from .session import Session
from .urls import BASE_URL
from .utils import Logger # noqa:F401
if sys.version_info >= (3, 7):
from asyncio import BoundedSemaphore
semaphore = BoundedSemaphore(value=50)
class ForgeBase(object):
"""
Superclass for all api model classes in this Forge Python Wrapper.
"""
session = Session(base_url=BASE_URL)
TODAY = date.today()
TODAY_STRING = TODAY.strftime("%Y-%m-%d")
IN_ONE_YEAR_STRING = (TODAY + timedelta(365)).strftime("%Y-%m-%d")
# TODO - Reorganise Extension Types (https://forge.autodesk.com/en/docs/data/v2/developers_guide/basics/#extension-types) # noqa: E501
BASE_TYPES = [
def __init__(self, iterator, n):
self.iterator = iter(iterator)
self.back_pressure = BoundedSemaphore(n)
def __init__(self):
self.msg_semaphore = BoundedSemaphore(
self._max_concurrency_of_semaphore)
self.msg_rwlock = RWLockWrite()
self.flood_rwlock = RWLockWrite()
self.pending_callbacks = set()
from asyncio import TimeoutError, sleep, BoundedSemaphore
import datetime
import logging
from typing import Optional, NamedTuple, Tuple
import async_timeout # type: ignore
from aiohttp import ClientSession, client_exceptions as exceptions
import tldextract
log = logging.getLogger(__name__)
MAX_CONCURRENT_REQUESTS = 100
TIMEOUT = 5
_SEMAPHORE = BoundedSemaphore(value=MAX_CONCURRENT_REQUESTS)
class FetchResponse(NamedTuple):
domain: str
scraped_at: datetime.datetime
adstxt_present: Optional[bool]
response: Tuple[str, ...]
async def fetch(domain: str, user_agent: str) -> FetchResponse:
"""Fetch a domain over http, check for validity and return.
Args
Domain (str): string domain to fetch.
Returns
user: _USER_LIKE) -> tuple[BoundedSemaphore, RWLockWrite, RWLockWrite]:
"""
:return: user_msg_semaphore, user_msg_rwlock, user_flood_rwlock
"""
return user_msg_semaphore(user), user_msg_rwlock(user), user_flood_rwlock(
user)
def user_pending_callbacks(user: _USER_LIKE) -> set:
return _user_bucket[user].pending_callbacks
# ----- web locks -----
_hostname_semaphore_bucket: defaultdict[str, BoundedSemaphore] = defaultdict(
partial(BoundedSemaphore, 5))
overall_web_semaphore = BoundedSemaphore(100)
def hostname_semaphore(url: str, parse: bool = True) -> BoundedSemaphore:
hostname = urlparse(url).hostname if parse else url
return _hostname_semaphore_bucket[hostname]
# Q: Why msg rwlock is needed?
#
# A: We are using an async telegram lib `telethon`. It sends messages asynchronously. Unfortunately,
# sending messages to telegram is not "thread safe". Yup, you heard me right. That's all telegram's fault!
#
# In detail: Telegram needs time to send an album msg (media group msg) and each medium in the album is actually
# a single media msg — telegram just displays them as an album. However, if we send multiple msgs (some of them
# are album msgs), telegram cannot ensure that an album msg won't be interrupted by another msg. There's an