def __init__(
self,
privkey: datatypes.PrivateKey,
context: BasePeerContext,
max_peers: int = DEFAULT_MAX_PEERS,
peer_info: BasePeerInfo = None,
token: CancelToken = None,
event_bus: Endpoint = None,
) -> None:
super().__init__(token)
if peer_info is None:
peer_info = NoopPeerInfo()
self.peer_info = peer_info
self.privkey = privkey
self.max_peers = max_peers
self.context = context
self.connected_nodes: Dict[Node, BasePeer] = {}
self._subscribers: List[PeerSubscriber] = []
self._event_bus = event_bus
# Restricts the number of concurrent connection attempts can be made
self._connection_attempt_lock = asyncio.BoundedSemaphore(
MAX_CONCURRENT_CONNECTION_ATTEMPTS)
if self.has_event_bus:
self.peer_backends = self.setup_peer_backends()
else:
self.logger.warning("No event bus configured for peer pool.")
self.peer_backends = ()
def __init__(self,
input_tube,
job_processor,
loop=None,
host='localhost',
port=11300,
reserve_timeout=3,
concurrency=1,
bury_failures=False):
super().__init__()
self.host = host
self.port = port
self.loop = loop
self.input_tube = input_tube
self.conn = None
self.job_processor = job_processor
self.reserve_timeout = reserve_timeout
self.reserve_task_timeout = (reserve_timeout * 2) + 1
self.bury_failures = bury_failures
self.semaphore = asyncio.BoundedSemaphore(concurrency)
self.conn_semaphore = asyncio.Semaphore(1)
def test_lock_by_with_statement(self):
loop = asyncio.new_event_loop() # don't use TestLoop quirks
self.set_event_loop(loop)
primitives = [
asyncio.Lock(loop=loop),
asyncio.Condition(loop=loop),
asyncio.Semaphore(loop=loop),
asyncio.BoundedSemaphore(loop=loop),
]
@asyncio.coroutine
def test(lock):
yield from asyncio.sleep(0.01, loop=loop)
self.assertFalse(lock.locked())
with self.assertWarns(DeprecationWarning):
with (yield from lock) as _lock:
self.assertIs(_lock, None)
self.assertTrue(lock.locked())
yield from asyncio.sleep(0.01, loop=loop)
self.assertTrue(lock.locked())
self.assertFalse(lock.locked())
for primitive in primitives:
loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
def __init__(self, config):
self.config = config
self.fnmatch_names = [
name for name in config if is_fnmatch_pattern(name)
]
self.sgx = self.config.getboolean(config.default_section, 'sgx')
self.loader = [
fspath(config.getpath(config.default_section, 'loader').resolve())
]
if self.sgx:
self.loader.append('SGX')
self.bindir = (config.getpath(config.default_section, 'ltproot') /
'testcases/bin')
# Running parallel tests under SGX is risky, see README.
# However, if user wanted to do that, we shouldn't stand in the way,
# just issue a warning.
processes = config.getint(
config.default_section,
'jobs',
fallback=(1 if self.sgx else len(os.sched_getaffinity(0))))
if self.sgx and processes != 1:
_log.warning(
'WARNING: SGX is enabled and jobs = %d (!= 1);'
' expect stability issues', processes)
self.semaphore = asyncio.BoundedSemaphore(processes)
self.queue = []
self.xml = etree.Element('testsuite')
self.time = 0
def __init__(self, args, env):
self._set_paths(args, env)
compat.makedirs(self.state_dir)
self.cache = cache.Cache(self.cache_dir)
self._tmp_root = os.path.join(self.state_dir, 'tmp')
compat.makedirs(self._tmp_root)
self.overrides = KeyVal(os.path.join(self.state_dir, 'overrides'),
self._tmp_root)
self.force = args.get('--force', False)
if args['--quiet'] and args['--verbose']:
raise PrintableError(
"Peru can't be quiet and verbose at the same time.")
self.quiet = args['--quiet']
self.verbose = args['--verbose']
# Use a semaphore (a lock that allows N holders at once) to limit the
# number of fetches that can run in parallel.
num_fetches = _get_parallel_fetch_limit(args)
self.fetch_semaphore = asyncio.BoundedSemaphore(num_fetches)
# Use locks to make sure the same cache keys don't get double fetched.
self.cache_key_locks = collections.defaultdict(asyncio.Lock)
# Use a different set of locks to make sure that plugin cache dirs are
# only used by one job at a time.
self.plugin_cache_locks = collections.defaultdict(asyncio.Lock)
self.display = get_display(args)
async def fetch_all(urls, loop):
sem = asyncio.BoundedSemaphore(CONCURRENT_CONNECTIONS)
async with aiohttp.ClientSession(loop=loop) as session:
results = await asyncio.gather(
*[fetch(sem, session, url) for url in urls])
df = pd.DataFrame(results)
return df.loc[~df["JSON_Value"].str.contains(ERROR_MSG)]
def __init__(self, aio_loop):
self._aio_loop = aio_loop
self._meta_executor = ThreadPoolExecutor(max_workers=MAX_POOL_SIZE)
self._download_executors = defaultdict(partial(ThreadPoolExecutor,
max_workers=MAX_HOST_POOL_SIZE))
self._max_workers_sem = asyncio.BoundedSemaphore(value=MAX_POOL_SIZE)
self._interrupted = False
def __init__(self,
wrapper,
coro_fun,
iterables,
loop,
timeout=None,
limit=None,
out_of_order=False):
self._out_of_order = out_of_order
self._wrapper = wrapper
self._loop = loop
self._end_time = None
if timeout is not None:
self._end_time = time.time() + timeout
if limit is not None and limit > 0:
self._sema = asyncio.BoundedSemaphore(value=limit, loop=self._loop)
else:
self._sema = None
self._coro_fun = coro_fun
self._running_futures = set()
self._last_future = asyncio.Future(loop=self._loop)
self._running_futures.add(self._last_future)
self._async_result_queue = []
self._wakeup_iterator = asyncio.Condition(loop=self._loop)
self._iterables = iterables
self._stopped = False
self._generator = self._coroutine_generator()
def __init__(self):
self.logger = logging.getLogger("proxy_py/processor")
if settings.DEBUG:
self.logger.setLevel(logging.DEBUG)
else:
self.logger.setLevel(logging.INFO)
logger_file_handler = logging.FileHandler("logs/processor.log")
logger_file_handler.setLevel(logging.DEBUG)
logger_file_handler.setFormatter(logging.Formatter(LOGGERS_FORMAT))
self.logger.addHandler(logger_file_handler)
self.collectors_logger = logging.getLogger("proxy_py/collectors")
if settings.DEBUG:
self.collectors_logger.setLevel(logging.DEBUG)
else:
self.collectors_logger.setLevel(logging.INFO)
collectors_logger_file_handler = logging.FileHandler("logs/collectors.log")
collectors_logger_file_handler.setLevel(logging.DEBUG)
logger_file_handler.setFormatter(logging.Formatter(LOGGERS_FORMAT))
self.collectors_logger.addHandler(collectors_logger_file_handler)
self.logger.debug("processor initialization...")
self.proxies_semaphore = asyncio.BoundedSemaphore(settings.NUMBER_OF_CONCURRENT_TASKS)
self.good_proxies_are_processed = False
async def producer_fixture():
# Run a processing task and capture the metadata results in a mock kafka
# producer
redis = FakeRedis()
stats = StatsManager(redis)
meta_producer = FakeProducer()
retry_producer = FakeProducer()
producer = AsyncProducer(meta_producer, 'foo')
await process_image(
persister=validate_thumbnail,
session=RateLimitedClientSession(FakeAioSession(), redis),
url='https://example.gov/hello.jpg',
identifier='4bbfe191-1cca-4b9e-aff0-1d3044ef3f2d',
stats=stats,
source='example',
semaphore=asyncio.BoundedSemaphore(1000),
metadata_producer=producer,
retry_producer=retry_producer
)
producer_task = asyncio.create_task(producer.listen())
try:
await asyncio.wait_for(producer_task, 0.01)
except concurrent.futures.TimeoutError:
pass
return meta_producer, retry_producer
async def test_handled_404s():
redis = FakeRedis()
stats = StatsManager(redis)
kafka = FakeProducer()
rot_producer = AsyncProducer(kafka, 'foo')
session = RateLimitedClientSession(
FakeAioSession(corrupt=True, status=404), redis
)
ident = '4bbfe191-1cca-4b9e-aff0-1d3044ef3f2d'
await process_image(
persister=validate_thumbnail,
session=session,
url='fake_url',
identifier=ident,
stats=stats,
source='example',
semaphore=asyncio.BoundedSemaphore(1000),
rot_producer=rot_producer
)
producer_task = asyncio.create_task(rot_producer.listen())
try:
await asyncio.wait_for(producer_task, 0.01)
except concurrent.futures.TimeoutError:
pass
rot_msg = kafka.messages[0]
parsed = json.loads(str(rot_msg, 'utf-8'))
assert ident == parsed['identifier']
async def test_records_errors():
redis = FakeRedis()
stats = StatsManager(redis)
session = RateLimitedClientSession(FakeAioSession(status=403), redis)
retry_producer = FakeProducer()
producer = AsyncProducer(retry_producer, 'foo')
await process_image(
persister=validate_thumbnail,
session=session,
url='https://example.gov/image.jpg',
identifier='4bbfe191-1cca-4b9e-aff0-1d3044ef3f2d',
stats=stats,
source='example',
semaphore=asyncio.BoundedSemaphore(1000),
retry_producer=producer
)
expected_keys = [
'resize_errors',
'resize_errors:example',
'resize_errors:example:403',
'status60s:example',
'status1hr:example',
'status12hr:example'
]
for key in expected_keys:
val = redis.store[key]
assert val == 1 or len(val) == 1
producer_task = asyncio.create_task(producer.listen())
try:
await asyncio.wait_for(producer_task, 0.01)
except concurrent.futures.TimeoutError:
pass
retry = retry_producer.messages[0]
parsed = json.loads(str(retry, 'utf-8'))
assert parsed['attempts'] == 1
async def schedule_loop(self):
""" Repeatedly schedule image processing tasks. """
task_schedule = defaultdict(list)
semaphore = asyncio.BoundedSemaphore(settings.MAX_TASKS)
while True:
to_schedule = await self._schedule(task_schedule)
for source in to_schedule:
# Cull finished tasks
running = []
for task in task_schedule[source]:
if not task.done():
running.append(task)
task_schedule[source] = running
# Add new tasks
if to_schedule[source]:
log.info(f'Scheduling {len(to_schedule[source])} '
f'{source} downloads')
for msg in to_schedule[source]:
t = asyncio.create_task(
self.image_processor(url=msg['url'],
identifier=msg['uuid'],
source=source,
semaphore=semaphore))
task_schedule[source].append(t)
await asyncio.sleep(5)
async def fullscan(db, folders=None, concurrency=1, crawl=False):
if folders is None:
folders = await db.folders()
folders = [f['name'] for f in folders]
with click_spinner.spinner():
files = [
f for f in find_files(list(folders))
if f[1].endswith(tuple(supported_formats))
]
size = len(files) * 2 if crawl else len(files)
with tqdm(total=size,
file=sys.stdout,
desc="Loading music",
leave=True,
position=0,
disable=config.quiet) as bar:
async def insert(semaphore, f):
async with semaphore:
try:
m = File(f[1], f[0])
if crawl:
await m.find_youtube()
bar.update(1)
debug(m.to_list())
await db.upsert(m)
bar.update(1)
except asyncpg.exceptions.CheckViolationError as e:
warning("Violation: {}".format(e))
semaphore = asyncio.BoundedSemaphore(concurrency)
tasks = [asyncio.ensure_future(insert(semaphore, f)) for f in files]
await asyncio.gather(*tasks)
await db.refresh()
async def fetch_wiki_links_batch(session,
ids,
*,
num_simult=10,
batch_size=50):
lock = asyncio.BoundedSemaphore(num_simult)
try:
with open(_WBACKUP) as f:
data = json.load(f)
except FileNotFoundError:
data = {}
fetch_ids = list(set(ids).difference(data))
try:
await asyncio.gather(*[
fetch_wiki_links(session, lock, fetch_ids[i * batch_size:(i + 1) *
batch_size], data)
for i in range(-(-len(fetch_ids) // batch_size))
])
except:
logging.warning('writing wiki backup: %s', _WBACKUP)
with open(_WBACKUP, 'w') as f:
json.dump(data, f)
raise
return [data[wid] for wid in ids]
async def _run():
if num_jobs == 1:
# Special case for --jobs=1
for _case, dbname, setup_script in setup:
await _setup_database(dbname, setup_script, conn)
if verbose:
print(f' -> {dbname}: OK', flush=True)
else:
async with taskgroup.TaskGroup(name='setup test cases') as g:
# Use a semaphore to limit the concurrency of bootstrap
# tasks to the number of jobs (bootstrap is heavy, having
# more tasks than `--jobs` won't necessarily make
# things faster.)
sem = asyncio.BoundedSemaphore(num_jobs)
async def controller(coro, dbname, *args):
async with sem:
await coro(dbname, *args)
if verbose:
print(f' -> {dbname}: OK', flush=True)
for _case, dbname, setup_script in setup:
g.create_task(
controller(_setup_database, dbname, setup_script,
conn))
async def parse(*ids, num_simult=10):
lock = asyncio.BoundedSemaphore(num_simult)
async with aiohttp.ClientSession() as session:
futures = [fetch_id(session, lock, mgp_id) for mgp_id in ids]
for future in futures:
json.dump(await future, sys.stdout)
sys.stdout.write('\n')
def crawl_nodes_api(self, page_limit=0):
"""
The runner method that runs parse_nodes_api(), which will populate the list of self.node_url_tuples.
After self.node_url_tuples is populated, _truncate_node_url_tuples() will be called to truncate the list
according to self.date_modified_marker.
:param page_limit: Number of pages of API to crawl. If page_limit=0, then crawl all pages.
"""
sem = asyncio.BoundedSemaphore(value=10)
self.debug_logger.info("\nStart crawling nodes API pages")
# Request number of pages in nodes API
with requests.Session() as s:
response = s.get(self.api_base + 'nodes/' + '?filter[date_modified][gte]=' + self.date_modified_marker.isoformat(sep='T'))
j = response.json()
num_pages = math.ceil(j['links']['meta']['total'] / j['links']['meta']['per_page'])
if num_pages < page_limit or page_limit == 0:
page_limit = num_pages
tasks = []
for i in range(1, page_limit + 1):
tasks.append(asyncio.ensure_future(self.parse_nodes_api(
self.api_base + 'nodes/' + '?filter[date_modified][gte]=' +
self.date_modified_marker.isoformat(sep='T') +
'&page=' + str(i),
sem
)))
loop = asyncio.get_event_loop()
loop.run_until_complete(self._wait_with_progress_bar(tasks))
self.debug_logger.info("Finished crawling nodes API pages")
def __init__(self, queue_size, max_pods, parallelism=1):
self.queue_size = queue_size
self.queue = asyncio.Queue(maxsize=queue_size)
self.semaphore = asyncio.BoundedSemaphore(max_pods)
self.pending_pods = set()
self.created_pods = set()
workers = [
asyncio.ensure_future(self._create_pod())
for _ in range(parallelism)
]
async def manager(workers):
while True:
failed, pending = await asyncio.wait(
workers, return_when=asyncio.FIRST_EXCEPTION)
for fut in failed:
err = fut.exception()
assert err is not None
err_msg = '\n'.join(
traceback.format_exception(type(err), err,
err.__traceback__))
log.error(f'restarting failed worker: {err} {err_msg}')
pending.add(asyncio.ensure_future(self._create_pod()))
workers = pending
asyncio.ensure_future(manager(workers))
def crawl_institutions_api(self, page_limit=0):
"""
The runner method that runs parse_institutions_api(), which will populate the list of self.institution_urls.
:param page_limit: Number of pages of API to crawl. If page_limit=0, then crawl all pages.
"""
self.debug_logger.info("Start crawling institution API pages")
# Setting a semaphore for rate limiting
sem = asyncio.BoundedSemaphore(value=10)
self.debug_logger.info('\nStart crawling institution API pages')
# Request number of pages in nodes API
with requests.Session() as s:
response = s.get(self.api_base + 'institutions/')
j = response.json()
num_pages = math.ceil(j['links']['meta']['total'] / j['links']['meta']['per_page'])
if j['links']['meta']['total'] == 0:
print("No institutions.")
return
if num_pages < page_limit or page_limit == 0:
page_limit = num_pages
tasks = []
for i in range(1, page_limit + 1):
tasks.append(asyncio.ensure_future(self.parse_institutions_api(
self.api_base + 'institutions/?page=' + str(i),
sem
)))
loop = asyncio.get_event_loop()
loop.run_until_complete(self._wait_with_progress_bar(tasks))
self.debug_logger.info("Finished crawling institution API pages")
def __init__( # pylint: disable=too-many-arguments
self,
game,
api,
game_id,
sleep_time,
simultaneous_obs,
max_scheduled,
obs_memory,
obs_time,
):
super().__init__(game.role_names, game.strat_names,
game.num_role_players)
self._api = api
self._game = paygame.samplegame_copy(rsgame.empty_copy(game))
self._game_id = game_id
self._sleep_time = sleep_time
self._obs_memory = obs_memory
self._obs_time = obs_time
self._simult_obs = simultaneous_obs
self._is_open = False
self._profiles = {}
self._prof_ids = {}
self._sched = None
self._fetcher = None
self._sched_lock = asyncio.Lock()
self._scheduled = asyncio.BoundedSemaphore(max_scheduled *
simultaneous_obs)
def async_make_requests(requestPool, maxRequests=3):
# Duck test approach
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
semaphore = asyncio.BoundedSemaphore(maxRequests)
# tasks = asyncio.gather (*[make_request (**request) for request in requestPool])
tasks = asyncio.gather(*[(make_request_async(semaphore, **request))
for request in requestPool])
# for request in requestPool:
# tasks.append (asyncio.ensure_future (make_request (semaphore, **request)))
results = loop.run_until_complete(tasks)
responses = []
errors = []
for response in results:
errors.extend(response["errors"])
if len(response["errors"]) == 0:
responses.append(response["response"])
return {"responses": responses, "errors": errors}
def __init__(self) -> None:
self.appdirs = AppDirs(appname=__app_name__, roaming=True)
self.saved_accounts = Accounts(self)
self.ui_settings = UISettings(self)
self.ui_state = UIState(self)
self.history = History(self)
self.models = ModelStore()
self.clients: Dict[str, MatrixClient] = {}
self._sso_server: Optional[SSOServer] = None
self._sso_server_task: Optional[asyncio.Future] = None
self.profile_cache: Dict[str, nio.ProfileGetResponse] = {}
self.get_profile_locks: DefaultDict[str, asyncio.Lock] = \
DefaultDict(asyncio.Lock) # {user_id: lock}
self.send_locks: DefaultDict[str, asyncio.Lock] = \
DefaultDict(asyncio.Lock) # {room_id: lock}
cache_dir = Path(
os.environ.get("MIRAGE_CACHE_DIR") or self.appdirs.user_cache_dir,
)
self.media_cache: MediaCache = MediaCache(self, cache_dir)
self.presences: Dict[str, Presence] = {}
self.concurrent_get_presence_limit = asyncio.BoundedSemaphore(8)
self.mxc_events: DefaultDict[str, List[Event]] = DefaultDict(list)
def __init__(self):
self.proxy_storage = []
# 50 this is concurrent connections limit
self.async_semaphore = asyncio.BoundedSemaphore(50)
# 10800 == 3 hour
self.proxy_db = StorageProxies(10800)
self._min_working_proxies = 100
self._check_timeout = 30
self.trust_urls = (
'http://ebay.com/',
'http://facebook.com/',
'http://google.co.uk/',
'http://google.com/',
'http://google.de/',
'http://google.es/',
'http://google.fr/',
'http://google.it/',
'http://google.pl/',
'http://google.ru/',
'http://instagram.com/',
'http://mail.ru/',
'http://microsoft.com/',
'http://office.com/',
'http://ok.ru/',
'http://twitter.com/',
'http://vk.com/',
'http://wikipedia.org/',
'http://ya.ru/',
'http://yandex.ru/',
'http://youtube.com/',
)
def __init__(self):
self.logger = logging.getLogger("proxy_py/processor")
self.logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
logger_handler = logging.StreamHandler(sys.stdout)
logger_handler.setLevel(
logging.DEBUG if settings.DEBUG else logging.INFO)
logger_handler.setFormatter(
logging.Formatter(settings.LOG_FORMAT_STRING))
self.logger.addHandler(logger_handler)
self.collectors_logger = logging.getLogger("proxy_py/collectors")
self.collectors_logger.setLevel(
logging.DEBUG if settings.DEBUG else logging.INFO)
collectors_logger_handler = logging.StreamHandler(sys.stdout)
collectors_logger_handler.setLevel(
logging.DEBUG if settings.DEBUG else logging.INFO)
collectors_logger_handler.setFormatter(
logging.Formatter(settings.LOG_FORMAT_STRING))
self.collectors_logger.addHandler(collectors_logger_handler)
self.logger.debug("processor initialization...")
self.proxies_semaphore = asyncio.BoundedSemaphore(
settings.NUMBER_OF_CONCURRENT_TASKS)
self.good_proxies_are_processed = False
async def get_stash_tabs():
semaphore = asyncio.BoundedSemaphore(10)
params: dict = {}
async with aiohttp.ClientSession(cookies={c: s}) as session:
# async with session.head(f'{u}/character-window/get-stash-items') as r:
# rate_policy = {k: v for k, v in zip(['limit', 'interval', 'timeout'],
# r.headers['X-Rate-Limit-Account'].split(',')[0].split(":"))}
# rate_state = {k: v for k, v in zip(['limit', 'interval', 'timeout'],
# r.headers['X-Rate-Limit-Account-State'].split(',')[0].split(":"))}
async with session.get(f"{u}/character-window/get-account-name") as r:
params.update(await r.json())
params.update({"league": "Standard"})
async with session.get(f"{u}/character-window/get-stash-items",
params={
**{
"tabs": 1
},
**params
}) as r:
# for t in (await r.json())['tabs']:
# tabs.replace_one({'id': t['id']}, t, {'upsert': True})
tabs.insert_multiple((await r.json())["tabs"])
# tabs_to_get = (i['i'] for i in tabs.find({'type': {'$in': ['NormalStash', 'PremiumStash', 'QuadStash']}}))
tabs_to_get = (i["i"] for i in tabs.search(
Tab.type.one_of(["NormalStash", "PremiumStash", "QuadStash"])))
await asyncio.gather(*(get_tab(session, t, params, semaphore)
for t in tabs_to_get))
def __init__(self,
start_url: Union[List[str], str],
max_crawl_size=None,
timeout=30,
user_agent=None,
verbose=True,
**kwargs):
self._start_url = start_url
self._client: Union[None, ClientSession] = None
self._client_timeout: Union[None, ClientTimeout] = None
self._timeout: Union[int, None] = None
self._proxy_manager: \
Union[None, AbstractProxyManager] = kwargs.get('proxy_manager')(**kwargs) if kwargs.get('proxy_manager') else None
self._url_filter = self._set_url_filter(start_url, **kwargs)
self._task_queue = NewJobQueue(max_crawl_size, start_url)
self._executor = kwargs.get('executor', None)
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
self._logger = kwargs.get('logger', logging.getLogger("Scraper"))
self._client_timeout = self._setup_timeout_rules(timeout)
self.verbose = verbose
self._rate_limiter = RateLimiter(
kwargs.get('rate_limit')) if kwargs.get('rate_limit') else None
self.__remaining_coroutines = 0
self.__user_agent = user_agent
self._creation_semaphore = asyncio.BoundedSemaphore(1)
def __init__(
self,
rearq: "ReArq",
queue: Optional[str] = None,
group_name: Optional[str] = None,
consumer_name: Optional[str] = None,
):
self.group_name = group_name or socket.gethostname()
self.consumer_name = consumer_name
self.job_timeout = rearq.job_timeout
self.max_jobs = rearq.max_jobs
self.rearq = rearq
self._redis = rearq.redis
self._lock_manager = Aioredlock([(rearq.redis_host, rearq.redis_port)])
self.register_tasks = rearq.get_queue_tasks(queue)
if queue:
self.queue = QUEUE_KEY_PREFIX + queue
else:
self.queue = DEFAULT_QUEUE
self.loop = asyncio.get_event_loop()
self.sem = asyncio.BoundedSemaphore(self.max_jobs)
self.queue_read_limit = max(self.max_jobs * 5, 100)
self.tasks: Set[asyncio.Task[Any]] = set()
self._task_map = rearq.task_map
self.jobs_complete = 0
self.jobs_retried = 0
self.jobs_failed = 0
self.job_retry = rearq.job_retry
self.job_retry_after = rearq.job_retry_after
self._add_signal_handler(signal.SIGINT, self.handle_sig)
self._add_signal_handler(signal.SIGTERM, self.handle_sig)
self.rearq.create_task(True, check_pending_msgs, queue, "* * * * *")
def __init__(
self,
privkey: datatypes.PrivateKey,
context: BasePeerContext,
max_peers: int = DEFAULT_MAX_PEERS,
token: CancelToken = None,
event_bus: EndpointAPI = None,
) -> None:
super().__init__(token)
self.privkey = privkey
self.max_peers = max_peers
self.context = context
self.connected_nodes: Dict[SessionAPI, BasePeer] = {}
self._subscribers: List[PeerSubscriber] = []
self._event_bus = event_bus
# Restricts the number of concurrent connection attempts can be made
self._connection_attempt_lock = asyncio.BoundedSemaphore(
MAX_CONCURRENT_CONNECTION_ATTEMPTS)
# Ensure we can only have a single concurrent handshake in flight per remote
self._handshake_locks = ResourceLock()
self.peer_backends = self.setup_peer_backends()
self.connection_tracker = self.setup_connection_tracker()
def test_lock_by_with_statement(self):
loop = asyncio.new_event_loop() # don't use TestLoop quirks
self.set_event_loop(loop)
primitives = [
asyncio.Lock(),
asyncio.Condition(),
asyncio.Semaphore(),
asyncio.BoundedSemaphore(),
]
with self.assertWarns(DeprecationWarning):
@asyncio.coroutine
def test(lock):
yield from asyncio.sleep(0.01)
self.assertFalse(lock.locked())
with self.assertRaisesRegex(TypeError,
"object is not iterable"):
with (yield from lock):
pass
self.assertFalse(lock.locked())
for primitive in primitives:
loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
