def greetings(bot, mask, target, data=None, **kw):
# log.msg("Shortening URL %s" % url)
if bot.nick == mask.nick:
return
url = None
for match in __URL_REGEX.finditer(data):
url = match.group(0)
break
if not url:
return
"""Load configuration"""
config = bot.config.get(__name__, {})
with aiohttp.Timeout(10):
with aiohttp.ClientSession(loop=bot.loop) as session:
headers = {"Content-Type": "application/json"}
payload = {"url": url, "key": None}
resp = yield from session.post(__URL_KRZUS,
data=json.dumps(payload),
headers=headers)
if resp.status != 200:
raise Exception
r = yield from resp.read()
data = json.loads(r.decode('utf-8'))
short = data.get("url_short")
if not url:
return
desc = None
if url.endswith(__IMAGE_EXT) and ClarifaiApi is not None:
api = ClarifaiApi(config['clarifai_app_id'],
config['clarifai_app_secret'])
tags = api.tag_image_urls(url)
if tags['status_code'] == "OK":
ret = ', '.join(tags['results'][0]['result']['tag']['classes'])
desc = "Tags: {}".format(ret)
elif BeautifulSoup is not None:
with aiohttp.Timeout(10):
with aiohttp.ClientSession(loop=bot.loop) as session:
resp = yield from session.get(url)
if resp.status == 200:
r = yield from resp.read()
soup = BeautifulSoup(r, "html.parser")
title = soup.title.getText()
if len(title) > 60:
title = title[:60] + "…"
desc = "Title: {}".format(title)
if desc:
bot.privmsg(target, desc)
bot.privmsg(target, "\tURL: {}".format(short))
else:
bot.privmsg(target, "URL: {}".format(short))
async def setavatar(cmd, message, args):
if message.author.id in permitted_id:
loop = asyncio.get_event_loop()
aiosession = aiohttp.ClientSession(loop=loop)
url = ''.join(args)
try:
if message.attachments:
thing = message.attachments[0]['url']
else:
thing = url.strip('<>')
try:
with aiohttp.Timeout(10):
async with aiosession.get(thing) as res:
await cmd.bot.edit_profile(avatar=await res.read())
except Exception as e:
cmd.log.error(e)
return
except AttributeError:
try:
thing = ''.join(args)
try:
with aiohttp.Timeout(10):
async with aiosession.get(thing) as res:
await cmd.bot.edit_profile(avatar=await res.read())
except:
return
except ResourceWarning:
pass
except Exception as e:
await cmd.bot.send_message(message.channel, e)
def fetch(self, request_type, url, params, data):
"""Fetch one URL"""
tries = 0
exception = None
while tries < self.max_tries:
try:
print("try %s---->%d times" % (url, tries))
with aiohttp.Timeout(self.timeout):
response = yield from self.session.get(url, params=params)
if response.status == 200:
content_type = response.headers.get('content-type')
if content_type in CONTENT_TYPE_TEXT:
with aiohttp.Timeout(self.timeout):
content = yield from response.text(
encoding='GBK')
else:
with aiohttp.Timeout(self.timeout):
content = yield from response.read()
break
except asyncio.TimeoutError:
print("timeout")
except aiohttp.ClientError as client_error:
print("client error")
except Exception:
print("unknown error")
tries += 1
else:
print("try %s---->more than %d times, quit" % (url, tries))
return None
response.release()
return content
async def setavatar(cmd, message, args):
if message.author.id in permitted_id:
loop = asyncio.get_event_loop()
aiosession = aiohttp.ClientSession(loop=loop)
url = ''.join(args)
try:
if message.attachments:
thing = message.attachments[0]['url']
else:
thing = url.strip('<>')
try:
with aiohttp.Timeout(10):
async with aiosession.get(thing) as res:
await cmd.bot.edit_profile(avatar=await res.read())
embed = discord.Embed(
title=':white_check_mark: New Avatar Set',
color=0x66CC66)
await cmd.bot.send_message(message.channel,
None,
embed=embed)
except Exception as e:
cmd.log.error(e)
return
except AttributeError:
try:
thing = ''.join(args)
try:
with aiohttp.Timeout(10):
async with aiosession.get(thing) as res:
await cmd.bot.edit_profile(avatar=await res.read())
embed = discord.Embed(
title=':white_check_mark: New Avatar Set',
color=0x66CC66)
await cmd.bot.send_message(message.channel,
None,
embed=embed)
except:
return
except ResourceWarning:
pass
except Exception as e:
embed = discord.Embed(color=0xDB0000)
embed.add_field(name=':exclamation: Error', value=str(e))
await cmd.bot.send_message(message.channel, None, embed=embed)
else:
out = discord.Embed(
type='rich',
color=0xDB0000,
title=':no_entry: Insufficient Permissions. Bot Owner Only.')
await cmd.bot.send_message(message.channel, None, embed=out)
async def get_html_from_url(self, url: str) -> str:
with aiohttp.Timeout(self.timeout):
logger.debug("Fetching %s", url)
response = await aiohttp.get(url)
assert response.status == 200
html = await response.text()
return html
def test_recv_timeout(loop, test_client):
@asyncio.coroutine
def handler(request):
ws = web.WebSocketResponse()
yield from ws.prepare(request)
yield from ws.receive_str()
yield from asyncio.sleep(0.1, loop=request.app.loop)
yield from ws.close()
return ws
app = web.Application(loop=loop)
app.router.add_route('GET', '/', handler)
client = yield from test_client(app)
resp = yield from client.ws_connect('/')
resp.send_str('ask')
with pytest.raises(asyncio.TimeoutError):
with aiohttp.Timeout(0.01, loop=app.loop):
yield from resp.receive()
yield from resp.close()
def fetch(self, code, semaphore, proxy):
headers = {"User-Agent": "medoc1001119", "Host": "uakey.com.ua"}
url = 'http://uakey.com.ua/files/cert_list.php?edrpo=%s' % code
counter = 0
with (yield from semaphore):
while True:
counter += 1
if counter >= self.retry:
break
with aiohttp.Timeout(self.timeout):
try:
if self.proxies:
p = proxy.get_proxy
conn = aiohttp.ProxyConnector(proxy=p)
else:
conn = None
with aiohttp.ClientSession(connector=conn) as session:
response = yield from session.get(url,
headers=headers)
body = yield from response.read()
break
except Exception as err:
body = 'err'.encode('utf-8')
continue
return (code, body.decode('utf-8', errors='ignore'))
# proxy = []
# codes = ['35294300']
# a = UKeys(codes)
# res = a.start()
# print(res)
async def send(self, data, headers, timeout=None):
"""Use synchronous interface, because this is a coroutine."""
try:
with aiohttp.Timeout(timeout):
async with self.client.post(self._url,
data=data,
headers=headers) as response:
assert response.status == 202
except asyncio.TimeoutError as e:
print_trace = True
message = ("Connection to APM Server timed out "
"(url: %s, timeout: %s seconds)" % (self._url, timeout))
raise TransportException(message, data,
print_trace=print_trace) from e
except AssertionError as e:
print_trace = True
body = await response.read()
if response.status == 429:
message = 'Temporarily rate limited: '
print_trace = False
else:
message = 'Unable to reach APM Server: '
message += '%s (url: %s, body: %s)' % (e, self._url, body)
raise TransportException(message, data,
print_trace=print_trace) from e
except Exception as e:
print_trace = True
message = 'Unable to reach APM Server: %s (url: %s)' % (e,
self._url)
raise TransportException(message, data,
print_trace=print_trace) from e
else:
return response.headers.get('Location')
async def get(self, uri):
"""GET data using the REST API
Args:
uri: URI to get the data
Returns:
XML data string
Raises:
:class:`aiobosest.errors.CallError`: malformed requests
:class:`aiobosest.errors.RequestError`: request didn't receive HTTP 200
:class:`aiobosest.errors.RestConnectionError`: other errors"""
logging.debug('>>> REST URI: http://{address}:8090{uri}'.format(
address=self._address, uri=uri))
try:
with aiohttp.Timeout(HTTP_REST_TIMEOUT, loop=self._loop):
async with self._session.get('http://{address}:8090{uri}'.format(
address=self._address, uri=uri)) as resp:
data = await resp.read()
logging.debug('<<< RECEIVED FROM URI http://{address}:8090{uri}: {message}'
.format(address=self._address, uri=uri, message=data))
status = await self._status_xml_parse(data)
if status:
return data
else:
raise CallError
except CallError:
raise
except RequestError:
raise
except Exception as e:
logging.error('Error connecting to http://{address}:8090: {error!r}'.format(
address=self._address, error=e))
raise RESTConnectionError(e)
def loginout_pinbo(accountId):
url = "/player/logout"
data = {}
data['userCode'] = 'probet.' + accountId
headers = tokenhelp.gen_headers()
if procVariable.debug:
headers["refer"] = "probet"
session = aiohttp.ClientSession()
try:
with aiohttp.Timeout(10):
resp = yield from session.post(tokenhelp.PINBO_URL + url, headers=headers, data=data,verify_ssl=False)
if resp.status != 200:
logging.error("{}|{}".format(errorLogic.third_party_error[1],data))
raise exceptionLogic(errorLogic.third_party_error)
except Exception as e:
logging.exception(e)
raise exceptionLogic(errorLogic.sys_unknow_error)
else:
res = yield from resp.read()
resp = json.loads(res.decode())
finally:
if session is not None:
yield from session.close()
return resp
async def fetch_page():
with aiohttp.Timeout(10):
async with session.get(
'https://poloniex.com/public?command=return24hVolume'
) as response:
assert response.status == 200
return await response.read()
async def wait_for_status(batch_id, wait, request):
'''Wait until transaction status is not PENDING (COMMITTED or error).
'wait' is time to wait for status, in seconds.
'''
headers = {'Content-Type': 'application/json'}
# waited = 0
# start_time = time.time()
# wait = request.app.config.TIMEOUT
# # timeout = aiohttp.ClientTimeout(total=request.app.config.TIMEOUT)
# while waited < wait:
try:
with aiohttp.Timeout(request.app.config.TIMEOUT):
async with aiohttp.ClientSession() as session:
async with session.get(
"http://127.0.0.1:8008/batch_statuses?id={}&wait={}".
format(batch_id, wait),
headers=headers) as response:
# print(response)
await asyncio.sleep(4)
data = await response.read()
# print(data)
data = load_json(data)
# print(data)
status = data['data'][0]['status']
print(status)
except Exception as e:
# logger.error("Error in wait for status")
# logger.error(e)
status = ""
pass
async def send(data, request):
"""
batch_request = client_batch_submit_pb2.ClientBatchSubmitRequest()
batch_request.batches.extend(batches)
await conn.send(
validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST,
batch_request.SerializeToString(),
timeout)
"""
headers = {'Content-Type': 'application/octet-stream'}
# timeout = aiohttp.Timeout(request.app.config.TIMEOUT)
# print(timeout)
try:
with aiohttp.Timeout(request.app.config.TIMEOUT):
async with aiohttp.ClientSession() as session:
async with session.post("http://127.0.0.1:8008/batches",
data=data,
headers=headers) as response:
# print(response)
data = await response.read()
# print(data)
except Exception as e:
# logger.error("Blockchain rest-api is unreachable, Please fix it dude")
raise ApiInternalError(
"Blockchain rest-api is unreachable, Please fix it dude")
return data
async def get_account_state(address, request):
# try:
headers = {'Content-Type': 'application/json'}
with aiohttp.Timeout(request.app.config.TIMEOUT):
async with aiohttp.ClientSession() as session:
async with session.get(
"http://127.0.0.1:8008/state/{}".format(address),
headers=headers) as response:
# print(response)
await asyncio.sleep(4)
data = await response.read()
# print(type(data))
data = data.decode("utf-8")
# data =literal_eval(data)
data = json.loads(data)
# print(data["data"])
# data2 = base64.b64decode(data["data"])
# print(data2)
# except Exception as e:
# pass
return base64.b64decode(data["data"])
async def put_resource(self, api_path, data, **kwargs):
"""
Helper method for HTTP PUT API requests.
Args:
api_path(str): REST API path
data: JSON data for POST request
Keyword Args:
kwargs: keyword args used for replacing items in the API path
"""
put_headers = {
'Content-Type': 'application/json',
**self.headers
}
url = self.build_api_url(
api_path.format(
tenant=self.tenant,
controllerId=self.controller_id,
**kwargs))
self.logger.debug('PUT {}'.format(url))
self.logger.debug(json.dumps(data))
with aiohttp.Timeout(self.timeout):
async with self.session.put(url, headers=put_headers,
data=json.dumps(data)) as resp:
await self.check_http_status(resp)
async def __asyncgetnvalue(sessions, future, dictdata, index, num, *codes, timeout=30):
if len(codes) > 1010:
raise ValueError('Too many code input: %s' % len(codes))
fronturl = 'http://api.money.126.net/data/feed/'
backurl = ',money.api'
codelist = list()
def got_result(fut):
data = fut.result()
dictdata.update(json.loads(data[21:-2]))
num[0] -= 1
num[1] -= 1
logging.info('%s End,remain %s/%s' % (index, num[1], num[2]))
logging.info('__asyncgetnvalue:Input data number %s' % len(codes))
if len(codes) > 1010:
raise ValueError('Too many code input: %s' % len(codes))
for code in codes:
codelist.append(str(code))
url = fronturl + ','.join(codelist) + backurl
with aiohttp.Timeout(timeout):
async with sessions.get(url) as response:
logging.info('%s Running' % index)
num[0] += 1
future.set_result(await response.text())
logging.info('now running %s fur' % num[0])
future.add_done_callback(got_result)
def pageget(url):
part = url.split("/")
relative_dir = part[3] + "/" + part[5] + "/" + part[6] + "/"
filename = part[7][:-3] + "html"
headers = {
"User-Agent":
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36"
}
with aiohttp.ClientSession() as session:
with aiohttp.Timeout(5):
response = yield from session.get(url, headers=headers)
try:
data = yield from response.text("utf-8")
data = re.sub("http://blog.nogizaka46.com/",
r'\.\./\.\./\.\./\.\./', data)
if os.path.exists(base_dir + relative_dir) == 0:
os.makedirs(base_dir + relative_dir)
print(filename)
f = open(base_dir + relative_dir + filename, "w")
f.write(data)
f.close()
return
except BaseException as e:
print(e)
return
async def fetch_page(session, url):
with aiohttp.Timeout(10):
# Now, we are sending our HTTP request (GET method) to the HTTP server.
# Even it takes too much time, await keyword will successfully "yield" computing resource to other functions.
async with session.get(url) as response:
assert response.status == 200
return await response.read()
async def _get_img(self, session, url):
image_name = self._get_image_name(url)
with aiohttp.Timeout(10):
try:
async with session.get(url) as response:
if response.status == 200:
img = await response.read()
try:
with open(image_name, 'wb') as f:
f.write(img)
except Exception as why:
log.error("Error storing fetched file from url "
"{}: {}. "
"Skipping this url.".format(url, why))
else:
self._fetched_imgs.append(image_name)
self._print_progress_bar(len(self._fetched_imgs),
len(self))
else:
print("Can't fetch image "
"{}: {}".format(url, response))
except Exception as why:
log.error("Error fetching url '{}': '{}'... "
"Skipping this url.".format(url[:50],
str(why)[:100]))
self._len -= 1
async def __anext__(self):
"""
Decode each line using json
Returns
-------
dict
Decoded JSON data
"""
line = b''
try:
while not line:
with aiohttp.Timeout(self._timeout):
line = await self.response.content.readline()
line = line.rstrip(b'\r\n')
if line in rate_limit_notices:
raise StreamLimit(line)
return self.loads(line)
except StreamLimit:
return await self.restart_stream(error=True)
except StopAsyncIteration:
return await self.restart_stream(error=True)
except json.decoder.JSONDecodeError:
return await self.restart_stream(error=True)
except asyncio.TimeoutError:
return await self.restart_stream(reconnect=0, error=True)
except aiohttp.errors.ContentEncodingError:
return await self.restart_stream(reconnect=0, error=True)
async def queue_message(self, channel_id: str, msg):
embed = '0'
if type(msg) == discord.Embed:
embed = '1'
msg = jsonpickle.encode(msg)
else:
msg = str(msg)
message_id = random.randint(0, 1000000)
payload = {
'key': 'verysecretkey',
'id': message_id,
'channel_id': channel_id,
'message': msg,
'embed': embed
}
try:
with aiohttp.Timeout(15):
async with self.session.post('http://ip:port/queue',
data=payload) as r:
return True
except (asyncio.TimeoutError, aiohttp.errors.ClientConnectionError,
aiohttp.errors.ClientError):
await asyncio.sleep(5)
return
except Exception as e:
print('queue error: ' + str(e))
async def call(self, endpoint, method='POST', raw=False, *args, **kwargs):
if 'headers' not in kwargs:
kwargs['headers'] = await self.get_headers()
uri = self.uri(endpoint)
logger.debug('Fetching: %s', uri)
logger.debug('Headers: %s', kwargs['headers'])
logger.debug('Cookies: %s', self.session.cookies)
with aiohttp.Timeout(self.request_timeout):
async with self.session.request(method, uri, *args,
**kwargs) as response:
body = await response.read()
if not response.status == 200:
try:
json = await response.json()
except Exception: # TODO: narrow exception
json = None
ex = BadRequest if response.status == 400 else HTTPError
raise ex(response.status, body, kwargs.get('data'), json)
if raw:
return body
json = await response.json()
if json.get('error'):
raise ResponseError(response.status, body,
kwargs.get('data'), json)
return json
def handleHttp(request: dict):
# 获取用户列表接口 get
url = tokenhelp.PINBO_URL + '/list-player/info'
headers = tokenhelp.gen_headers()
session = aiohttp.ClientSession()
try:
with aiohttp.Timeout(10):
resp = session.get(url=url, headers=headers, verify_ssl=False)
if resp.status != 200:
raise exceptionLogic(errorLogic.client_param_invalid)
except Exception as e:
logging.exception(repr(e))
raise exceptionLogic(errorLogic.sys_unknow_error)
else:
res = yield from resp.read()
res = json.loads(res.decode())
code = res.get('code', '')
if code != '' and (code in errorLogic.pinbo_error_code.keys()):
logging.debug(code + ":" + errorLogic.pinbo_error_code[code])
raise exceptionLogic([code, errorLogic.pinbo_error_code[code]])
return res
finally:
if session:
yield from session.close()
def fetch(session, url):
with aiohttp.Timeout(10):
resp = yield from session.get(url)
try:
return (yield from resp.text())
except:
yield from resp.release()
async def get(self, address, params, headers):
with aiohttp.Timeout(self.operation_timeout):
response = await self.session.get(address,
params=params,
headers=headers)
return await self.new_response(response)
async def fetch_page(session, url):
with aiohttp.Timeout(10):
async with session.get(url) as response:
assert response.status == 200
content = await response.read()
print(response.status, content[0:60])
return content
async def get_header(session, url, headerfield=None, *, timeout=5):
with aiohttp.Timeout(timeout):
async with session.head(url) as response:
if headerfield:
return response.headers.get(headerfield)
else:
return response.headers
async def fetch(session, coin):
params = {"fsym": coin, "tsyms": fetch_price_params}
with aiohttp.Timeout(fetch_timeout):
async with session.get(cc_url, params=params) as response:
prices = await response.json()
result = {"CC_price": prices, "CC_symbol": coin, "Time": get_epoch_time()}
return result
async def downloadImage(url, folder, name, loop, chunkSize=20):
result = {'canAccessURL': False, 'isImage': False, 'fileSaved': False}
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'
}
async with aiohttp.ClientSession(loop=loop) as session:
with aiohttp.Timeout(10, loop=session.loop):
async with session.get(url, headers=headers) as response:
content_type = response.headers['content-type']
if response.status == 200:
result['canAccessURL'] = True
if "image" in content_type:
result['isImage'] = True
if not result['canAccessURL'] or not result['isImage']:
return result
extension = mimetypes.guess_extension(content_type)
if extension == '.jpe':
extension = '.jpg'
with open(folder + "/" + name + extension, 'wb') as fd:
while True:
chunk = await response.content.read(chunkSize)
if not chunk:
break
fd.write(chunk)
result['fileSaved'] = True
return result
def test_auth_with_valid_data(self):
s = TestAuthSession(login=USER_LOGIN,
password=USER_PASSWORD,
app_id=APP_ID)
s.driver.session = aiohttp.ClientSession(
connector=aiohttp.TCPConnector(verify_ssl=False),
response_class=CustomClientResponse)
yield from s.authorize()
params = {
'client_id': APP_ID,
'display': 'page',
'redirect_uri': REDIRECT_URI,
'response_type': 'code'
}
with aiohttp.Timeout(10):
response = yield from s.driver.session.get(
"https://oauth.vk.com/authorize",
params=params,
allow_redirects=True)
s.close()
code = response.url.query.get('code')
self.assertIsNotNone(code)
s = AuthorizationCodeSession(APP_ID, APP_SECRET, REDIRECT_URI, code)
yield from s.authorize()
s.close()
self.assertIsNotNone(s.access_token)
