def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.semaphore = asyncio.Semaphore(kwargs.get('in_parallel', 1))
def test_initial_value_zero(self): with self.assertWarns(DeprecationWarning): sem = asyncio.Semaphore(0, loop=self.loop) self.assertTrue(sem.locked())
""" Elasticsearch Query Execution """ import asyncio from biothings.web.handlers.exceptions import BadRequest, EndRequest from elasticsearch import (ConnectionError, ConnectionTimeout, NotFoundError, RequestError, TransportError) from elasticsearch_dsl import MultiSearch from tornado.web import HTTPError semaphore = asyncio.Semaphore() class ESQueryBackend(object): ''' Execute an Elasticsearch query ''' def __init__(self, web_settings): self.client = web_settings.connections.async_client # es indices self.indices = web_settings.ES_INDICES self.default_index = web_settings.ES_INDEX self.default_type = web_settings.ES_DOC_TYPE # for scroll queries self.scroll_time = web_settings.ES_SCROLL_TIME self.scroll_size = web_settings.ES_SCROLL_SIZE
try: mydirpath = pathlib.Path(__file__).parent except NameError: mydirpath = pathlib.Path(".") _delay = None def _delay_f(): h = datetime.datetime.utcnow().hour if h in range(2, 10): return 5 elif h in range(10, 20): return 10 else: return 20 _sem = asyncio.Semaphore(1) async def _sem_release(loop): await asyncio.sleep(_delay_f(), loop=loop) _sem.release() # The objects below are mutable! # Never change them without copying first! with (mydirpath / "clientReplay.json").open() as f: clientReplay = json.load(f) with (mydirpath / "clientCloseSession.json").open() as f: clientCloseSession = json.load(f) def obj2msg(obj): _json = json.dumps(obj, separators=JSON_SEPARATORS)
import uvloop from lxml import html logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger('douban') USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36' proxies = { 'https': 'http://spin.printf.me:3128', } UserMeta = namedtuple('UserMeta', 'home_url name') asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.get_event_loop() sema = asyncio.Semaphore(15, loop=loop) users = set() # proxies = requests.get('http://spin.printf.me:8000/?types=0&country=国内&count=10') # ip_ports = json.loads(proxies.text) # sema = asyncio.BoundedSemaphore(100) def get_data(filename, default=''): """ Get data from a file :param filename: filename :param default: default value :return: data """ try:
def __init__(self): self.semaphore = asyncio.Semaphore() self.loaded = False
def __init__(self, namespace): super().__init__(namespace) self.loop = asyncio.new_event_loop() self.semaphore = asyncio.Semaphore( models.get_settings()['process_count'], loop=self.loop)
async def run_sender(self): # print('---------------------') roomid = self._room_id wait_to_send_danmu = {} # 礼物列表合并后的输出 sem = asyncio.Semaphore(1) # print('-----run_sender') if self.user.disable_gift_thx: return try: while (1): # 取出所有结果,添加到等待队列 # 如果某个room-user-gift保持了5s不动,则推出 async with sem: qlength = self.GIFT_QUEUE.qsize() cache_gift = [] for i in range(qlength): cache_gift.append(self.GIFT_QUEUE.get()) # print(cache_gift) # cache_gift是所有没处理的送礼物的信息 # 现在将他们合并为一个list for gift_info in cache_gift: if gift_info.get('room') != roomid: print('error room id') exit(0) # 拿到单条礼物信息 username, gift_name, gift_num, t, coin_type, total_coin = gift_info.get( 'username'), gift_info.get('gift_name'), gift_info.get( 'gift_num'), gift_info.get('t'), gift_info.get( 'coin_type'), gift_info.get('total_coin') # 以用户名为主键 if username not in wait_to_send_danmu: wait_to_send_danmu[username] = {} # 新建username # 礼物名为主键 if f'{gift_name}_{coin_type}' not in wait_to_send_danmu.get( username): wait_to_send_danmu[username].update({ f'{gift_name}_{coin_type}': { 'gift_num': gift_num, 'coin_type': coin_type, 'total_coin': total_coin, 't': t, } }) # username->gift_name else: # 查找已经送了的有多少 already_num = wait_to_send_danmu[username].get( f'{gift_name}_{coin_type}', {}).get('gift_num', 0) # 已经送了的 already_total_coin = wait_to_send_danmu[username].get( f'{gift_name}_{coin_type}', {}).get('total_coin', 0) # 已经送了的总价值 wait_to_send_danmu[username][ f'{gift_name}_{coin_type}'].update({ 'gift_num': gift_num + already_num, 't': t, 'total_coin': total_coin + already_total_coin }) # 更新数量 # 检查时间是否达到推出标准 # 这里可以重写感谢弹幕 # print(wait_to_send_danmu) for username, gifts in wait_to_send_danmu.items(): for gift_name, info in gifts.items(): gift_num = info.get('gift_num') if gift_num == 0: continue coin_type = info.get('coin_type') total_coin = info.get('total_coin', 0) # flag_gold # print(gift_name) gift_name_true = gift_name.replace('_gold', '').replace( '_silver', '') fstr = '' if self.user.const_json.get( 'normal_gift_thx_format' ) and gift_name in self.user.const_json.get( 'normal_gift_thx_format'): fstr = self.user.const_json.get( 'normal_gift_thx_format').get(gift_name) elif self.user.const_json.get( 'normal_gift_thx_format' ) and gift_name_true in self.user.const_json.get( 'normal_gift_thx_format'): fstr = self.user.const_json.get( 'normal_gift_thx_format').get(gift_name_true) elif coin_type == 'silver': print(gift_name) self.write_log(f'{gift_name}没有设置') fstr = self.user.silver_gift_thx_format else: fstr = self.user.gold_gift_thx_format print(gift_name) self.write_log(f'{gift_name}没有设置') # print(fstr) if time.time() - info.get( 't') > self.user.gift_comb_delay: if self.is_live or (not self.user.only_live_thx): # self.user.gift_thx_silver_format await self.send_danmu( fstr.format(username=username, num=gift_num, total_coin=total_coin, giftname=gift_name_true, random1=random.choice( self.user.random_list_1), random2=random.choice( self.user.random_list_2), random3=random.choice( self.user.random_list_3))) await self.game_log(coin_type, total_coin) wait_to_send_danmu[username][gift_name].update({ 'gift_num': 0, 'total_coin': 0 }) await asyncio.sleep(0.5) except: traceback.print_exc()
def __init__(self, value=1): self._sema = asyncio.Semaphore(value=value)
import aiohttp import asyncio NUMBERS = list(range(12)) URL = 'http://httpbin.org/get?a={}' sema = asyncio.Semaphore(3) async def fetch_async(a): async with aiohttp.request('GET', URL.format(a)) as r: data = await r.json() return data['args']['a'] async def print_result(a): with (await sema): r = await fetch_async(a) print(('fetch({}) = {}'.format(a, r))) loop = asyncio.get_event_loop() f = asyncio.wait([print_result(num) for num in NUMBERS]) loop.run_until_complete(f)
async def test_full_body() -> None: body = Body(None, None) limit = 3 semaphore = asyncio.Semaphore(limit) asyncio.ensure_future(_fill_body(body, semaphore, limit)) assert b"012" == await body
async def create_task(start, end): semaphore = asyncio.Semaphore(10) # 限制并发量为500 for i in range(start, end): task = asyncio.ensure_future( get_response(url_template.format(i + 1), semaphore)) tasks.append(task)
MONGO_DB_NAME = 'books' MONGO_COLLECTION_NAME = 'books20200505' client = AsyncIOMotorClient(MONGO_CONNECTION_STRING) db = client[MONGO_DB_NAME] collection = db[MONGO_COLLECTION_NAME] # Scrape initialize PAGE_SIZE = 18 PAGE_NUMBER = 10 CONCURRENCY = 5 ids = [] INDEX_URL = 'https://dynamic5.scrape.cuiqingcai.com/api/book/?limit=18&offset={offset}' DETAIL_URL = 'https://dynamic5.scrape.cuiqingcai.com/api/book/{id}' logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s') semaphore = asyncio.Semaphore(CONCURRENCY) session = None # Scrape page by url async def scrape_api(url): async with semaphore: try: logging.info('scraping %s', url) async with session.get(url) as response: return await response.json() except aiohttp.ClientError: logging.error('error occurred while scraping %s', url, exc_info=True)
def CerberoSemaphore(value=1): get_event_loop() # this ensures the proper event loop is already created return asyncio.Semaphore(value)
async it "passes on limit if it has acquire", V: a = mock.Mock(name="a") limit = mock.NonCallableMock(name="limit", spec=["acquire"]) kwargs = {"b": a, "limit": limit} sender = mock.NonCallableMock(name="sender") async with SenderWrapper(V.target, sender, kwargs) as result: assert result is sender assert kwargs == {"b": a, "limit": limit} assert V.called == [] async it "passes on limit if it is already a Semaphore", V: a = mock.Mock(name="a") limit = asyncio.Semaphore(1) kwargs = {"b": a, "limit": limit} sender = mock.NonCallableMock(name="sender") async with SenderWrapper(V.target, sender, kwargs) as result: assert result is sender assert kwargs == {"b": a, "limit": limit} assert V.called == [] async it "creates and closes the sender if none provided", V: a = mock.Mock(name="a") kwargs = {"b": a} async with SenderWrapper(V.target, sb.NotSpecified, kwargs) as sender: assert sender is V.sender
async def start(self): # Invariant: semaphore._value == limit - open - _n_connecting self.semaphore = asyncio.Semaphore(self.limit)
def __init__(self, working_directory, concurrency=2): self.working_directory = working_directory self.semaphore = asyncio.Semaphore(concurrency) self.ongoing_tasks = {}
def func(args): in_df = None out_df = None test_column = UTTERANCE_COLUMN if args.test_column is not None: # Test input has multiple columns test_column = args.test_column in_df = pd.read_csv(args.infile, quoting=csv.QUOTE_ALL, encoding=UTF_8, keep_default_na=False) if test_column not in in_df: # Look for target test_column raise ValueError( "Test column {} doesn't exist in file.".format(test_column)) if args.merge_input: # Merge rest of columns from input to output out_df = in_df else: out_df = in_df[[test_column]].copy() out_df.columns = [test_column] else: test_series = pd.read_csv(args.infile, quoting=csv.QUOTE_ALL, encoding=UTF_8, header=None, squeeze=True, keep_default_na=False) if isinstance(test_series, pd.DataFrame): raise ValueError('Unknown test column') # Test input has only one column and no header out_df = test_series.to_frame() out_df.columns = [test_column] # Initial columns for test output for column in test_out_header: out_df[column] = '' # Applied coroutines sem = asyncio.Semaphore(args.rate_limit) loop = asyncio.get_event_loop() authenticator = choose_auth(args) nlc = NaturalLanguageClassifierV1(authenticator=authenticator) nlc.set_service_url(args.url) tasks = (fill_df(out_df.loc[row_idx, test_column], row_idx, out_df, args.workspace_id, nlc, sem) for row_idx in range(out_df.shape[0])) loop.run_until_complete(asyncio.gather(*tasks)) loop.close() if args.golden_intent_column is not None: golden_intent_column = args.golden_intent_column if golden_intent_column not in in_df.columns: print("No golden intent column '{}' is found in input.".format( golden_intent_column)) else: # Add INTENT_JUDGE_COLUMN based on golden_intent_column out_df[INTENT_JUDGE_COLUMN] = \ (in_df[golden_intent_column] == out_df[PREDICTED_INTENT_COLUMN]).map(BOOL_MAP) out_df[SCORE_COLUMN] = \ out_df[INTENT_JUDGE_COLUMN].map({'yes': 1, 'no': 0}) if args.partial_credit_table is not None: credit_tables = parse_partial_credit_table(args.partial_credit_table) for row_idx in range(out_df.shape[0]): golden_intent = out_df.loc[row_idx, args.golden_intent_column].strip() predict_intent = out_df.loc[row_idx, PREDICTED_INTENT_COLUMN].strip() if golden_intent == predict_intent: out_df.loc[row_idx, SCORE_COLUMN] = 1.0 elif golden_intent not in credit_tables or \ predict_intent not in credit_tables[golden_intent]: out_df.loc[row_idx, SCORE_COLUMN] = 0 else: out_df.loc[row_idx, SCORE_COLUMN] = \ credit_tables[golden_intent][predict_intent] save_dataframe_as_csv(df=out_df, file=args.outfile) print("Wrote test result file to {}".format(args.outfile))
def __init__(self, *args, **kwargs): super(NotificationSession, self).__init__(*args, **kwargs) self.subscriptions = defaultdict(list) self.cache = {} self.in_flight_requests_semaphore = asyncio.Semaphore(100)
def __init__(self, max_concurrent): self._require_non_negative(max_concurrent) self._max_concurrent = max_concurrent self.semaphore = asyncio.Semaphore(max_concurrent)
def __init__(self, maxsize=0): self.count = 0 self.limiter = asyncio.Semaphore(maxsize) self.empty = asyncio.Lock()
def __init__(self, **kwargs): super().__init__(**kwargs) # if self._debug: self._allow_state_change = asyncio.Semaphore(0) self._current_status_wait_future = None
'/api/browse?productCategory=sneakers&sort=deadstock_sold&order=DESC&page=', 'volatility': '/api/browse?productCategory=sneakers&sort=deadstock_sold&order=DESC&page=', 'pricePremiun': '/api/browse?order=DESC&productCategory=sneakers&sort=price_premium&page=', 'lastSale': '/api/browse?order=DESC&productCategory=sneakers&sort=last_sale&page=', 'lowestAsk': '/api/browse?order=ASC&productCategory=sneakers&sort=lowest_ask&page=', 'highestBid': '/api/browse?order=DESC&productCategory=sneakers&sort=highest_bid&page=', 'releaseDate': '/api/browse?order=DESC&productCategory=sneakers&sort=release_date&page=', } sem = asyncio.Semaphore(500) async def getData(url): async with sem: try: async with aiohttp.ClientSession() as session: async with session.get(url, headers=conf.headers, timeout=60) as resp: if resp.status == 200: print('URL: ', url) ret_json = await resp.json() return ret_json except: print('[ERROR] URL: ', url)
def semaphore(self): return asyncio.Semaphore()
def test_ctor_noloop(self): asyncio.set_event_loop(self.loop) sem = asyncio.Semaphore() self.assertIs(sem._loop, self.loop)
def test_acquire(self): sem = asyncio.Semaphore(3, loop=self.loop) result = [] self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertFalse(sem.locked()) @asyncio.coroutine def c1(result): yield from sem.acquire() result.append(1) return True @asyncio.coroutine def c2(result): yield from sem.acquire() result.append(2) return True @asyncio.coroutine def c3(result): yield from sem.acquire() result.append(3) return True @asyncio.coroutine def c4(result): yield from sem.acquire() result.append(4) return True t1 = asyncio.Task(c1(result), loop=self.loop) t2 = asyncio.Task(c2(result), loop=self.loop) t3 = asyncio.Task(c3(result), loop=self.loop) test_utils.run_briefly(self.loop) self.assertEqual([1], result) self.assertTrue(sem.locked()) self.assertEqual(2, len(sem._waiters)) self.assertEqual(0, sem._value) t4 = asyncio.Task(c4(result), loop=self.loop) sem.release() sem.release() self.assertEqual(2, sem._value) test_utils.run_briefly(self.loop) self.assertEqual(0, sem._value) self.assertEqual(3, len(result)) self.assertTrue(sem.locked()) self.assertEqual(1, len(sem._waiters)) self.assertEqual(0, sem._value) self.assertTrue(t1.done()) self.assertTrue(t1.result()) race_tasks = [t2, t3, t4] done_tasks = [t for t in race_tasks if t.done() and t.result()] self.assertTrue(2, len(done_tasks)) # cleanup locked semaphore sem.release() self.loop.run_until_complete(asyncio.gather(*race_tasks))
def test_acquire(self): with self.assertWarns(DeprecationWarning): sem = asyncio.Semaphore(3, loop=self.loop) result = [] self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertFalse(sem.locked()) async def c1(result): await sem.acquire() result.append(1) return True async def c2(result): await sem.acquire() result.append(2) return True async def c3(result): await sem.acquire() result.append(3) return True async def c4(result): await sem.acquire() result.append(4) return True t1 = self.loop.create_task(c1(result)) t2 = self.loop.create_task(c2(result)) t3 = self.loop.create_task(c3(result)) test_utils.run_briefly(self.loop) self.assertEqual([1], result) self.assertTrue(sem.locked()) self.assertEqual(2, len(sem._waiters)) self.assertEqual(0, sem._value) t4 = self.loop.create_task(c4(result)) sem.release() sem.release() self.assertEqual(2, sem._value) test_utils.run_briefly(self.loop) self.assertEqual(0, sem._value) self.assertEqual(3, len(result)) self.assertTrue(sem.locked()) self.assertEqual(1, len(sem._waiters)) self.assertEqual(0, sem._value) self.assertTrue(t1.done()) self.assertTrue(t1.result()) race_tasks = [t2, t3, t4] done_tasks = [t for t in race_tasks if t.done() and t.result()] self.assertTrue(2, len(done_tasks)) # cleanup locked semaphore sem.release() self.loop.run_until_complete(asyncio.gather(*race_tasks))
def test_initial_value_zero(self): sem = asyncio.Semaphore(0, loop=self.loop) self.assertTrue(sem.locked())
LON_MIN = -74.036351 LAT_DEG_P_M = (40.727740 - 40.726840) / 100 LON_DEG_P_M = (-73.978720 - -73.979907) / 100 # Define the radius (in meters) for the search RADIUS = 100 SPACING = 2 * RADIUS / math.sqrt(2) LAT_STEP = SPACING * LAT_DEG_P_M LON_STEP = SPACING * LON_DEG_P_M LAT_CALLS = int(math.floor((LAT_MAX - LAT_MIN + LAT_STEP) / (LAT_STEP))) LON_CALLS = int(math.floor((LON_MAX - LON_MIN + LON_STEP) / (LON_STEP))) sem = asyncio.Semaphore(20) place_ref_dict = {} def init_locations(): location_list = [] for i in range(LAT_CALLS): cur_lat = LAT_MIN + i * LAT_STEP for j in range(LON_CALLS): cur_lon = LON_MIN + j * LON_STEP cur_str = '{:.6f},{:.6f}'.format(cur_lat, cur_lon) location_list.append(cur_str) logging.info("total number of locations to query %d" % len(location_list)) return location_list
def __init__(self, value: int): self._semaphore = asyncio.Semaphore(value)