def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.semaphore = asyncio.Semaphore(kwargs.get('in_parallel', 1))
def test_initial_value_zero(self):
with self.assertWarns(DeprecationWarning):
sem = asyncio.Semaphore(0, loop=self.loop)
self.assertTrue(sem.locked())
"""
Elasticsearch Query Execution
"""
import asyncio
from biothings.web.handlers.exceptions import BadRequest, EndRequest
from elasticsearch import (ConnectionError, ConnectionTimeout, NotFoundError,
RequestError, TransportError)
from elasticsearch_dsl import MultiSearch
from tornado.web import HTTPError
semaphore = asyncio.Semaphore()
class ESQueryBackend(object):
'''
Execute an Elasticsearch query
'''
def __init__(self, web_settings):
self.client = web_settings.connections.async_client
# es indices
self.indices = web_settings.ES_INDICES
self.default_index = web_settings.ES_INDEX
self.default_type = web_settings.ES_DOC_TYPE
# for scroll queries
self.scroll_time = web_settings.ES_SCROLL_TIME
self.scroll_size = web_settings.ES_SCROLL_SIZE
try:
mydirpath = pathlib.Path(__file__).parent
except NameError:
mydirpath = pathlib.Path(".")
_delay = None
def _delay_f():
h = datetime.datetime.utcnow().hour
if h in range(2, 10):
return 5
elif h in range(10, 20):
return 10
else:
return 20
_sem = asyncio.Semaphore(1)
async def _sem_release(loop):
await asyncio.sleep(_delay_f(), loop=loop)
_sem.release()
# The objects below are mutable!
# Never change them without copying first!
with (mydirpath / "clientReplay.json").open() as f:
clientReplay = json.load(f)
with (mydirpath / "clientCloseSession.json").open() as f:
clientCloseSession = json.load(f)
def obj2msg(obj):
_json = json.dumps(obj, separators=JSON_SEPARATORS)
import uvloop
from lxml import html
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('douban')
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'
proxies = {
'https': 'http://spin.printf.me:3128',
}
UserMeta = namedtuple('UserMeta', 'home_url name')
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
sema = asyncio.Semaphore(15, loop=loop)
users = set()
# proxies = requests.get('http://spin.printf.me:8000/?types=0&country=国内&count=10')
# ip_ports = json.loads(proxies.text)
# sema = asyncio.BoundedSemaphore(100)
def get_data(filename, default=''):
"""
Get data from a file
:param filename: filename
:param default: default value
:return: data
"""
try:
def __init__(self):
self.semaphore = asyncio.Semaphore()
self.loaded = False
def __init__(self, namespace):
super().__init__(namespace)
self.loop = asyncio.new_event_loop()
self.semaphore = asyncio.Semaphore(
models.get_settings()['process_count'], loop=self.loop)
async def run_sender(self):
# print('---------------------')
roomid = self._room_id
wait_to_send_danmu = {} # 礼物列表合并后的输出
sem = asyncio.Semaphore(1)
# print('-----run_sender')
if self.user.disable_gift_thx:
return
try:
while (1):
# 取出所有结果，添加到等待队列
# 如果某个room-user-gift保持了5s不动，则推出
async with sem:
qlength = self.GIFT_QUEUE.qsize()
cache_gift = []
for i in range(qlength):
cache_gift.append(self.GIFT_QUEUE.get())
# print(cache_gift)
# cache_gift是所有没处理的送礼物的信息
# 现在将他们合并为一个list
for gift_info in cache_gift:
if gift_info.get('room') != roomid:
print('error room id')
exit(0)
# 拿到单条礼物信息
username, gift_name, gift_num, t, coin_type, total_coin = gift_info.get(
'username'), gift_info.get('gift_name'), gift_info.get(
'gift_num'), gift_info.get('t'), gift_info.get(
'coin_type'), gift_info.get('total_coin')
# 以用户名为主键
if username not in wait_to_send_danmu:
wait_to_send_danmu[username] = {} # 新建username
# 礼物名为主键
if f'{gift_name}_{coin_type}' not in wait_to_send_danmu.get(
username):
wait_to_send_danmu[username].update({
f'{gift_name}_{coin_type}': {
'gift_num': gift_num,
'coin_type': coin_type,
'total_coin': total_coin,
't': t,
}
}) # username->gift_name
else:
# 查找已经送了的有多少
already_num = wait_to_send_danmu[username].get(
f'{gift_name}_{coin_type}', {}).get('gift_num',
0) # 已经送了的
already_total_coin = wait_to_send_danmu[username].get(
f'{gift_name}_{coin_type}',
{}).get('total_coin', 0) # 已经送了的总价值
wait_to_send_danmu[username][
f'{gift_name}_{coin_type}'].update({
'gift_num':
gift_num + already_num,
't':
t,
'total_coin':
total_coin + already_total_coin
}) # 更新数量
# 检查时间是否达到推出标准
# 这里可以重写感谢弹幕
# print(wait_to_send_danmu)
for username, gifts in wait_to_send_danmu.items():
for gift_name, info in gifts.items():
gift_num = info.get('gift_num')
if gift_num == 0:
continue
coin_type = info.get('coin_type')
total_coin = info.get('total_coin', 0)
# flag_gold
# print(gift_name)
gift_name_true = gift_name.replace('_gold',
'').replace(
'_silver', '')
fstr = ''
if self.user.const_json.get(
'normal_gift_thx_format'
) and gift_name in self.user.const_json.get(
'normal_gift_thx_format'):
fstr = self.user.const_json.get(
'normal_gift_thx_format').get(gift_name)
elif self.user.const_json.get(
'normal_gift_thx_format'
) and gift_name_true in self.user.const_json.get(
'normal_gift_thx_format'):
fstr = self.user.const_json.get(
'normal_gift_thx_format').get(gift_name_true)
elif coin_type == 'silver':
print(gift_name)
self.write_log(f'{gift_name}没有设置')
fstr = self.user.silver_gift_thx_format
else:
fstr = self.user.gold_gift_thx_format
print(gift_name)
self.write_log(f'{gift_name}没有设置')
# print(fstr)
if time.time() - info.get(
't') > self.user.gift_comb_delay:
if self.is_live or (not self.user.only_live_thx):
# self.user.gift_thx_silver_format
await self.send_danmu(
fstr.format(username=username,
num=gift_num,
total_coin=total_coin,
giftname=gift_name_true,
random1=random.choice(
self.user.random_list_1),
random2=random.choice(
self.user.random_list_2),
random3=random.choice(
self.user.random_list_3)))
await self.game_log(coin_type, total_coin)
wait_to_send_danmu[username][gift_name].update({
'gift_num':
0,
'total_coin':
0
})
await asyncio.sleep(0.5)
except:
traceback.print_exc()
def __init__(self, value=1):
self._sema = asyncio.Semaphore(value=value)
import aiohttp
import asyncio
NUMBERS = list(range(12))
URL = 'http://httpbin.org/get?a={}'
sema = asyncio.Semaphore(3)
async def fetch_async(a):
async with aiohttp.request('GET', URL.format(a)) as r:
data = await r.json()
return data['args']['a']
async def print_result(a):
with (await sema):
r = await fetch_async(a)
print(('fetch({}) = {}'.format(a, r)))
loop = asyncio.get_event_loop()
f = asyncio.wait([print_result(num) for num in NUMBERS])
loop.run_until_complete(f)
async def test_full_body() -> None:
body = Body(None, None)
limit = 3
semaphore = asyncio.Semaphore(limit)
asyncio.ensure_future(_fill_body(body, semaphore, limit))
assert b"012" == await body
async def create_task(start, end):
semaphore = asyncio.Semaphore(10) # 限制并发量为500
for i in range(start, end):
task = asyncio.ensure_future(
get_response(url_template.format(i + 1), semaphore))
tasks.append(task)
MONGO_DB_NAME = 'books'
MONGO_COLLECTION_NAME = 'books20200505'
client = AsyncIOMotorClient(MONGO_CONNECTION_STRING)
db = client[MONGO_DB_NAME]
collection = db[MONGO_COLLECTION_NAME]
# Scrape initialize
PAGE_SIZE = 18
PAGE_NUMBER = 10
CONCURRENCY = 5
ids = []
INDEX_URL = 'https://dynamic5.scrape.cuiqingcai.com/api/book/?limit=18&offset={offset}'
DETAIL_URL = 'https://dynamic5.scrape.cuiqingcai.com/api/book/{id}'
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s: %(message)s')
semaphore = asyncio.Semaphore(CONCURRENCY)
session = None
# Scrape page by url
async def scrape_api(url):
async with semaphore:
try:
logging.info('scraping %s', url)
async with session.get(url) as response:
return await response.json()
except aiohttp.ClientError:
logging.error('error occurred while scraping %s',
url,
exc_info=True)
def CerberoSemaphore(value=1):
get_event_loop() # this ensures the proper event loop is already created
return asyncio.Semaphore(value)
async it "passes on limit if it has acquire", V:
a = mock.Mock(name="a")
limit = mock.NonCallableMock(name="limit", spec=["acquire"])
kwargs = {"b": a, "limit": limit}
sender = mock.NonCallableMock(name="sender")
async with SenderWrapper(V.target, sender, kwargs) as result:
assert result is sender
assert kwargs == {"b": a, "limit": limit}
assert V.called == []
async it "passes on limit if it is already a Semaphore", V:
a = mock.Mock(name="a")
limit = asyncio.Semaphore(1)
kwargs = {"b": a, "limit": limit}
sender = mock.NonCallableMock(name="sender")
async with SenderWrapper(V.target, sender, kwargs) as result:
assert result is sender
assert kwargs == {"b": a, "limit": limit}
assert V.called == []
async it "creates and closes the sender if none provided", V:
a = mock.Mock(name="a")
kwargs = {"b": a}
async with SenderWrapper(V.target, sb.NotSpecified, kwargs) as sender:
assert sender is V.sender
async def start(self):
# Invariant: semaphore._value == limit - open - _n_connecting
self.semaphore = asyncio.Semaphore(self.limit)
def __init__(self, working_directory, concurrency=2):
self.working_directory = working_directory
self.semaphore = asyncio.Semaphore(concurrency)
self.ongoing_tasks = {}
def func(args):
in_df = None
out_df = None
test_column = UTTERANCE_COLUMN
if args.test_column is not None: # Test input has multiple columns
test_column = args.test_column
in_df = pd.read_csv(args.infile,
quoting=csv.QUOTE_ALL,
encoding=UTF_8,
keep_default_na=False)
if test_column not in in_df: # Look for target test_column
raise ValueError(
"Test column {} doesn't exist in file.".format(test_column))
if args.merge_input: # Merge rest of columns from input to output
out_df = in_df
else:
out_df = in_df[[test_column]].copy()
out_df.columns = [test_column]
else:
test_series = pd.read_csv(args.infile,
quoting=csv.QUOTE_ALL,
encoding=UTF_8,
header=None,
squeeze=True,
keep_default_na=False)
if isinstance(test_series, pd.DataFrame):
raise ValueError('Unknown test column')
# Test input has only one column and no header
out_df = test_series.to_frame()
out_df.columns = [test_column]
# Initial columns for test output
for column in test_out_header:
out_df[column] = ''
# Applied coroutines
sem = asyncio.Semaphore(args.rate_limit)
loop = asyncio.get_event_loop()
authenticator = choose_auth(args)
nlc = NaturalLanguageClassifierV1(authenticator=authenticator)
nlc.set_service_url(args.url)
tasks = (fill_df(out_df.loc[row_idx, test_column], row_idx, out_df,
args.workspace_id, nlc, sem)
for row_idx in range(out_df.shape[0]))
loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
if args.golden_intent_column is not None:
golden_intent_column = args.golden_intent_column
if golden_intent_column not in in_df.columns:
print("No golden intent column '{}' is found in input.".format(
golden_intent_column))
else: # Add INTENT_JUDGE_COLUMN based on golden_intent_column
out_df[INTENT_JUDGE_COLUMN] = \
(in_df[golden_intent_column]
== out_df[PREDICTED_INTENT_COLUMN]).map(BOOL_MAP)
out_df[SCORE_COLUMN] = \
out_df[INTENT_JUDGE_COLUMN].map({'yes': 1, 'no': 0})
if args.partial_credit_table is not None:
credit_tables = parse_partial_credit_table(args.partial_credit_table)
for row_idx in range(out_df.shape[0]):
golden_intent = out_df.loc[row_idx,
args.golden_intent_column].strip()
predict_intent = out_df.loc[row_idx,
PREDICTED_INTENT_COLUMN].strip()
if golden_intent == predict_intent:
out_df.loc[row_idx, SCORE_COLUMN] = 1.0
elif golden_intent not in credit_tables or \
predict_intent not in credit_tables[golden_intent]:
out_df.loc[row_idx, SCORE_COLUMN] = 0
else:
out_df.loc[row_idx, SCORE_COLUMN] = \
credit_tables[golden_intent][predict_intent]
save_dataframe_as_csv(df=out_df, file=args.outfile)
print("Wrote test result file to {}".format(args.outfile))
def __init__(self, *args, **kwargs):
super(NotificationSession, self).__init__(*args, **kwargs)
self.subscriptions = defaultdict(list)
self.cache = {}
self.in_flight_requests_semaphore = asyncio.Semaphore(100)
def __init__(self, max_concurrent):
self._require_non_negative(max_concurrent)
self._max_concurrent = max_concurrent
self.semaphore = asyncio.Semaphore(max_concurrent)
def __init__(self, maxsize=0):
self.count = 0
self.limiter = asyncio.Semaphore(maxsize)
self.empty = asyncio.Lock()
def __init__(self, **kwargs):
super().__init__(**kwargs)
# if self._debug:
self._allow_state_change = asyncio.Semaphore(0)
self._current_status_wait_future = None
'/api/browse?productCategory=sneakers&sort=deadstock_sold&order=DESC&page=',
'volatility':
'/api/browse?productCategory=sneakers&sort=deadstock_sold&order=DESC&page=',
'pricePremiun':
'/api/browse?order=DESC&productCategory=sneakers&sort=price_premium&page=',
'lastSale':
'/api/browse?order=DESC&productCategory=sneakers&sort=last_sale&page=',
'lowestAsk':
'/api/browse?order=ASC&productCategory=sneakers&sort=lowest_ask&page=',
'highestBid':
'/api/browse?order=DESC&productCategory=sneakers&sort=highest_bid&page=',
'releaseDate':
'/api/browse?order=DESC&productCategory=sneakers&sort=release_date&page=',
}
sem = asyncio.Semaphore(500)
async def getData(url):
async with sem:
try:
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=conf.headers,
timeout=60) as resp:
if resp.status == 200:
print('URL: ', url)
ret_json = await resp.json()
return ret_json
except:
print('[ERROR] URL: ', url)
def semaphore(self):
return asyncio.Semaphore()
def test_ctor_noloop(self):
asyncio.set_event_loop(self.loop)
sem = asyncio.Semaphore()
self.assertIs(sem._loop, self.loop)
def test_acquire(self):
sem = asyncio.Semaphore(3, loop=self.loop)
result = []
self.assertTrue(self.loop.run_until_complete(sem.acquire()))
self.assertTrue(self.loop.run_until_complete(sem.acquire()))
self.assertFalse(sem.locked())
@asyncio.coroutine
def c1(result):
yield from sem.acquire()
result.append(1)
return True
@asyncio.coroutine
def c2(result):
yield from sem.acquire()
result.append(2)
return True
@asyncio.coroutine
def c3(result):
yield from sem.acquire()
result.append(3)
return True
@asyncio.coroutine
def c4(result):
yield from sem.acquire()
result.append(4)
return True
t1 = asyncio.Task(c1(result), loop=self.loop)
t2 = asyncio.Task(c2(result), loop=self.loop)
t3 = asyncio.Task(c3(result), loop=self.loop)
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
self.assertTrue(sem.locked())
self.assertEqual(2, len(sem._waiters))
self.assertEqual(0, sem._value)
t4 = asyncio.Task(c4(result), loop=self.loop)
sem.release()
sem.release()
self.assertEqual(2, sem._value)
test_utils.run_briefly(self.loop)
self.assertEqual(0, sem._value)
self.assertEqual(3, len(result))
self.assertTrue(sem.locked())
self.assertEqual(1, len(sem._waiters))
self.assertEqual(0, sem._value)
self.assertTrue(t1.done())
self.assertTrue(t1.result())
race_tasks = [t2, t3, t4]
done_tasks = [t for t in race_tasks if t.done() and t.result()]
self.assertTrue(2, len(done_tasks))
# cleanup locked semaphore
sem.release()
self.loop.run_until_complete(asyncio.gather(*race_tasks))
def test_acquire(self):
with self.assertWarns(DeprecationWarning):
sem = asyncio.Semaphore(3, loop=self.loop)
result = []
self.assertTrue(self.loop.run_until_complete(sem.acquire()))
self.assertTrue(self.loop.run_until_complete(sem.acquire()))
self.assertFalse(sem.locked())
async def c1(result):
await sem.acquire()
result.append(1)
return True
async def c2(result):
await sem.acquire()
result.append(2)
return True
async def c3(result):
await sem.acquire()
result.append(3)
return True
async def c4(result):
await sem.acquire()
result.append(4)
return True
t1 = self.loop.create_task(c1(result))
t2 = self.loop.create_task(c2(result))
t3 = self.loop.create_task(c3(result))
test_utils.run_briefly(self.loop)
self.assertEqual([1], result)
self.assertTrue(sem.locked())
self.assertEqual(2, len(sem._waiters))
self.assertEqual(0, sem._value)
t4 = self.loop.create_task(c4(result))
sem.release()
sem.release()
self.assertEqual(2, sem._value)
test_utils.run_briefly(self.loop)
self.assertEqual(0, sem._value)
self.assertEqual(3, len(result))
self.assertTrue(sem.locked())
self.assertEqual(1, len(sem._waiters))
self.assertEqual(0, sem._value)
self.assertTrue(t1.done())
self.assertTrue(t1.result())
race_tasks = [t2, t3, t4]
done_tasks = [t for t in race_tasks if t.done() and t.result()]
self.assertTrue(2, len(done_tasks))
# cleanup locked semaphore
sem.release()
self.loop.run_until_complete(asyncio.gather(*race_tasks))
def test_initial_value_zero(self):
sem = asyncio.Semaphore(0, loop=self.loop)
self.assertTrue(sem.locked())
LON_MIN = -74.036351
LAT_DEG_P_M = (40.727740 - 40.726840) / 100
LON_DEG_P_M = (-73.978720 - -73.979907) / 100
# Define the radius (in meters) for the search
RADIUS = 100
SPACING = 2 * RADIUS / math.sqrt(2)
LAT_STEP = SPACING * LAT_DEG_P_M
LON_STEP = SPACING * LON_DEG_P_M
LAT_CALLS = int(math.floor((LAT_MAX - LAT_MIN + LAT_STEP) / (LAT_STEP)))
LON_CALLS = int(math.floor((LON_MAX - LON_MIN + LON_STEP) / (LON_STEP)))
sem = asyncio.Semaphore(20)
place_ref_dict = {}
def init_locations():
location_list = []
for i in range(LAT_CALLS):
cur_lat = LAT_MIN + i * LAT_STEP
for j in range(LON_CALLS):
cur_lon = LON_MIN + j * LON_STEP
cur_str = '{:.6f},{:.6f}'.format(cur_lat, cur_lon)
location_list.append(cur_str)
logging.info("total number of locations to query %d" % len(location_list))
return location_list
def __init__(self, value: int):
self._semaphore = asyncio.Semaphore(value)
