async def http_tester(self, test_info):
'''
Method to test using HTTP or HTTPS using HTTP3 Library
HTTP3 has async capabilities
'''
test_info['last_stat'] = test_info['status']
test_info['rtt'] = '--'
test_info['good'] = False
test_info['status'] = 'Incomplete'
try:
client = http3.AsyncClient()
start = time.time()
if test_info['test'] == 'HTTP':
url = f'http://{test_info["host"]}'
resp = await client.get(url, timeout=self.timeout)
if test_info['test'] == 'HTTPS':
url = f'https://{test_info["host"]}'
resp = await client.get(url,
timeout=self.timeout,
verify=False)
test_info['good'] = True
test_info['status'] = self.stat_dict.get(resp.status_code,
'Unknown')
except http3.exceptions.RedirectLoop:
test_info['good'] = False
test_info['status'] = 'Redirect Loop'
except http3.exceptions.ConnectTimeout:
test_info['good'] = False
test_info['status'] = 'Time Out'
except http3.exceptions.ReadTimeout:
test_info['good'] = False
test_info['status'] = 'Time Out'
except socket.gaierror:
test_info['good'] = False
test_info['status'] = 'Bad Address'
except OSError:
test_info['good'] = False
test_info['status'] = 'Unreachable'
except asyncio.CancelledError:
print('Cancelled !!')
test_info['total'] += 1
if not test_info['good']:
test_info['last_bad'] = datetime.now().strftime('%a %H:%M:%S')
test_info['last_bad_status'] = test_info['status']
else:
test_info['last_good'] = datetime.now().strftime('%a %H:%M:%S')
test_info['total_successes'] += 1
test_info['rtt'] = '{:.2f} ms'.format((time.time() - start) * 1000)
test_info['success_percent'] = sucPer(test_info['total'],
test_info['total_successes'])
return test_info
async def create_txn(self):
"""
:return: 0 if txn is finished else 1
"""
# mock the transaction information
client = http3.AsyncClient(timeout=100)
txn_id = str(uuid.uuid4())
amount = random.randint(1, 130)
item = ["iPhone12", "MacBook Pro", "Mac Mini",
"iPad"][random.randint(0, 3)]
location = self.loc if random.random() < 0.8 else "FakePlace"
merchant_number = str(uuid.uuid4())
# print("txn: ", txn_id)
# grant the approval
ret = await client.post(BASE_URL,
json={
"transaction_id": txn_id,
"user_id": self.id,
"amount": amount,
"location": location
})
# print(ret)
# poll to check if we've gotten the approval
while (ret := await
client.get(f"{BASE_URL}/{txn_id}")).status_code != 200:
# print("waiting")
await asyncio.sleep(1)
async def get_account(client_url: str, address: str) -> AddressBalance:
"""Get account from address
:param client_url: The haskoin API url
:type client_url: str
:param address: The BCH address
:type address: str
:returns: AddressBalance
"""
try:
api_url = f'{client_url}/address/{address}/balance'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
balance_response = json.loads(response.content.decode('utf-8'))
if "error" in balance_response:
raise Exception(
f'Error is : {balance_response["error"]}\nMessage is : {balance_response["message"]}'
)
else:
result = AddressBalance(balance_response['received'],
balance_response['utxo'],
balance_response['address'],
balance_response['txs'],
balance_response['unconfirmed'],
balance_response['confirmed'])
return result
else:
return None
except Exception as err:
raise Exception(str(err))
async def get_balance(sochain_url: str, net: str, address: str):
"""Get address balance
https://sochain.com/api#get-balance
:param net: mainnet or testnet
:type net: str
:param address: wallet address
:type address: str
:returns: The fees with memo
"""
try:
api_url = f'{sochain_url}/get_address_balance/{to_sochain_network(net)}/{address}'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
balance_response = json.loads(
response.content.decode('utf-8'))['data']
confirmed = float(balance_response['confirmed_balance'])
unconfirmed = float(balance_response['unconfirmed_balance'])
total = confirmed + unconfirmed
return total
else:
return None
except Exception as err:
raise Exception(str(err))
async def get_suggested_tx_fee():
"""Get suggested fee amount for Bitcoin cash. (fee per byte)
Note: Haskcoin does not provide fee rate related data
So use Bitgo API for fee estimation
Refer: https://app.bitgo.com/docs/#operation/v2.tx.getfeeestimate
:returns: The Bitcoin cash stats
"""
try:
api_url = 'https://app.bitgo.com/api/v2/bch/tx/fee'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
fee_response = json.loads(response.content.decode('utf-8'))
return fee_response["feePerKb"] / 1000 # feePerKb to feePerByte
else:
fee_response = json.loads(response.content.decode('utf-8'))
if "error" in fee_response:
raise Exception(f'Error is : {fee_response["error"]}')
return utils.DEFAULT_SUGGESTED_TRANSACTION_FEE
except:
return utils.DEFAULT_SUGGESTED_TRANSACTION_FEE
async def makeTheRequest(self, infoList):
client = http3.AsyncClient()
response = await client.get(
f"https://api.exchangeratesapi.io/latest?base={infoList[2].upper()}"
)
responseData = response.text
responseData = json.loads(responseData)
return responseData
async def test_access_content_stream_response(server):
async with http3.AsyncClient() as client:
response = await client.request("GET",
"http://127.0.0.1:8000/",
stream=True)
assert response.status_code == 200
with pytest.raises(http3.ResponseNotRead):
response.content
async def post_web_scrape_data(self, auth_token, web_scrape_data):
try:
msg = f"Inside post_web_scrape_data()=>auth_token={auth_token}"
logger.info(msg)
req_url = f"{app_settings.CLIENT_API_URL_PREFIX}brands"
msg = f"req_url:{req_url}"
logger.info(msg)
body_data = web_scrape_data
msg = {
"req_url": req_url,
"auth_token": auth_token,
"data": body_data
}
logger.info(msg)
headers_param = {
"Content-type": "application/json"
} if auth_token == "not_required" else {
"Authorization": f"Bearer {auth_token}",
"Content-type": "application/json"
}
msg = f"headers_param:{headers_param}"
logger.info(msg)
client = http3.AsyncClient()
post_data_batch_list = await self.get_data_batches(body_data)
batch_error_count: int = 0
for i in range(len(post_data_batch_list)):
try:
msg = f"Fetching data batch:{i + 1}"
logger.info(msg)
data_batch = post_data_batch_list[i]
# NOTE : If the list is empty then the string is '[]' which is a string of length 2
if data_batch is not None and len(data_batch) > 2:
msg = f"Data batch {i + 1} is valid"
logger.info(msg)
res = await client.post(url=req_url,
data=data_batch,
headers=headers_param)
logger.info(f"res:{res}")
msg = f"data_batch:{i + 1},res_status={res.status_code}, res-reason={res.reason_phrase}, res_text={res.text}"
logger.info(msg)
if res.status_code == 200:
batch_error_count += 0
else:
batch_error_count += 1
except Exception:
msg = f'WebScrapeHandler=>post_web_scrape_data():Data batch loop=>Data batch{i + 1}{sys.exc_info()[2]}/n{traceback.format_exc()} occurred'
logger.error(msg)
msg = f"dat_batch_error_count={batch_error_count}"
logger.info(msg)
result_message = "Publicó con éxito los datos del raspado de la web" if batch_error_count == 0 else f"{batch_error_count} de cada {len(post_data_batch_list)} lotes de datos no se cargaron"
result_status = "success" if batch_error_count == 0 else "error"
return result_status, result_message
except Exception:
msg = f'WebScrapeHandler=>post_web_scrape_data():{sys.exc_info()[2]}/n{traceback.format_exc()} occurred'
logger.error(msg)
return "error", "Se produjo un error al publicar los datos del web scrape"
async def test_get(server):
url = "http://127.0.0.1:8000/"
async with http3.AsyncClient() as client:
response = await client.get(url)
assert response.status_code == 200
assert response.text == "Hello, world!"
assert response.protocol == "HTTP/1.1"
assert response.headers
assert repr(response) == "<Response [200 OK]>"
async def test_stream_response(server):
async with http3.AsyncClient() as client:
response = await client.request("GET",
"http://127.0.0.1:8000/",
stream=True)
assert response.status_code == 200
body = await response.read()
assert body == b"Hello, world!"
assert response.content == b"Hello, world!"
async def test_stream_request(server):
async def hello_world():
yield b"Hello, "
yield b"world!"
async with http3.AsyncClient() as client:
response = await client.request("POST",
"http://127.0.0.1:8000/",
data=hello_world())
assert response.status_code == 200
async def test_100_continue(server):
url = "http://127.0.0.1:8000/echo_body"
headers = {"Expect": "100-continue"}
data = b"Echo request body"
async with http3.AsyncClient() as client:
response = await client.post(url, headers=headers, data=data)
assert response.status_code == 200
assert response.content == data
async def get_unspent_transactions(client_url, address) -> List[TxUnspent]:
"""Get unspent transactions
:param client_url: The haskoin API url
:type client_url: str
:param address: The BCH address
:type address: str
:returns: The Bitcoin cash stats
:raises: 'failed to query unspent transactions' if failed to query unspent transactions
"""
try:
account = await get_account(client_url, address)
api_url = f'{client_url}/address/{address}/unspent?limit={account.txs}'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
tx_response = json.loads(response.content.decode('utf-8'))
result = [
TxUnspent(i['pkscript'], i['value'], i['address'],
Block(i['block']['height'], i['block']['position']),
i['index'], i['txid']) for i in tx_response
]
return result
else:
raise Exception('failed to query unspent transactions')
except Exception as err:
raise Exception(str(err))
# async def broadcast_tx(client_url, tx_hex):
# """Broadcast transaction
# https://sochain.com/api#send-transaction
# :param client_url: The haskoin API url
# :type client_url: str
# :param tx_hex: tranaction hex
# :type tx_hex: str
# :returns: Transaction ID
# """
# try:
# api_url = f'{client_url}/transactions'
# client = http3.AsyncClient()
# response = await client.post(url=api_url, data=tx_hex)
# if response.status_code == 200:
# res = json.loads(response.content.decode('utf-8'))['data']
# return res['txid']
# else:
# return json.loads(response.content.decode('utf-8'))['data']
# except Exception as err:
# raise Exception(str(err))
async def worker(redis_uri: str,
rss_url: str,
rss_scrape_endpoint: str,
loop: asyncio.AbstractEventLoop = None):
""" pulls rss feeds from es-collector and creates a task for each of them
:param redis_uri: Redis connection endpoint
:param rss_url: es-rss endpoint
:param rss_scrape_endpoint: es-collector endpoint
:param loop: Eventloop on which this task should run
"""
loop = loop or asyncio.get_event_loop()
rss = Rss(rss_url)
redis = await aioredis.create_connection(redis_uri)
# Get all feeds and subscribed users from endpoint
async with http3.AsyncClient() as client:
resp = await client.request("GET", rss_scrape_endpoint)
if 200 != resp.status_code:
logger.warning(
f"Could not retrieve feeds from {rss_scrape_endpoint}")
parsed = ujson.loads(resp.text)
for feed in parsed:
# Get last parsed timestamp out of redis or assume it was scraped on 10.05.2019
last_time_raw = await redis.execute(
"get",
f"feed:{feed['link']}") or b"2019-05-10T00:00:00.000000+00:00"
last_time = datetime.fromisoformat(last_time_raw.decode("UTF-8"))
for job in await rss.get(feed["link"], last_time):
# Add to queue
await redis.execute(
"RPUSH",
"queue:items",
dumps(
QueueElement(
url=job.url,
title=job.title,
feed_url=feed["link"],
indexes=[f"user-{u['id']}"
for u in feed["users"]] # @TODO
)))
# Update timestamp
await redis.execute(
"set", f"feed:{feed['link']}",
datetime.now(timezone.utc).astimezone().isoformat())
logger.info(f"Parsed feed {feed['link']}")
async def test_raise_for_status(server):
async with http3.AsyncClient() as client:
for status_code in (200, 400, 404, 500, 505):
response = await client.request(
"GET", f"http://127.0.0.1:8000/status/{status_code}")
if 400 <= status_code < 600:
with pytest.raises(http3.exceptions.HttpError):
response.raise_for_status()
else:
assert response.raise_for_status() is None
async def push(self, to_push: PusherRequest) -> bool:
""" Pushes to the elasticsearch gateway """
async with http3.AsyncClient(timeout=None) as client:
resp = await client.request(
"POST",
self.pusher_url,
json=cvt_nested_datetime_isoformat(to_push))
return 200 == resp.status_code
return False
async def run_async_test2():
hues = [200,0]
dids=["80124473D31C5733151D95F44BA0C7711C5E8FB5",
"801246B0BCCAE659F048C513607844A61C5EAC18"]
get_token()
setup()
async with http3.AsyncClient() as client:
for hue in hues:
for di in dids:
url,data=make_hue_data(di,hue)
r = await client.post(url,json.dumps(data))
print(r)
def __init__(self, cookie=None):
self.requests = http3.AsyncClient()
self.cookies = {}
self.headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'en-US,en;q=0.5',
'Content-Type': 'application/json;charset=utf-8',
'Origin': 'https://www.roblox.com',
'X-CSRF-TOKEN': '',
'DNT': '1',
}
if cookie:
self.login(cookie)
async def get(self, feed_url: str,
from_time: datetime) -> List[RssResponse]:
""" Retrieves posts from an rss feed """
async with http3.AsyncClient(timeout=None) as client:
resp = await client.request("POST",
self.rss_url,
json={
"url": feed_url,
"from_time": from_time.isoformat()
})
if 200 != resp.status_code:
return []
# Parse json array into the correct container
return [RssResponse(**r) for r in ujson.loads(resp.text) or []]
async def call_http(ip: str):
client = http3.AsyncClient()
t = time.process_time()
try:
body = await call_api(ip)
resp = json.loads(body)
elapsed_time = time.process_time() - t
return {
"code": 0,
"type": "http",
"ip": ip,
"country": resp["country"],
"latency": elapsed_time
}
except Exception as e:
raise UnicornException(status=400, code=-20000, message=str(e))
async def generate_report(self) -> tuple:
client = http3.AsyncClient()
r = await client.post(url='https://icyftl.ru/ezgen/api/generate',
data=json.dumps(
{
'group_name': self.user.group,
'student_snp': self.user.student_snp,
'teacher_snp': self.user.teacher_snp,
'year': datetime.now().year,
'prac_number': self.prac_num,
'code': self.code,
'token': self.user.hash
},
ensure_ascii=False))
if r.status_code != 200:
return False, r.text
return True, r.content
async def scrape(self, url: str) -> ScrapeResponse:
""" Queries es-scraper for a specific url """
async with http3.AsyncClient(timeout=None) as client:
resp = await client.request("POST",
self.scrape_url,
json={"url": url})
try:
if 200 != resp.status_code:
return None
# Try to load resposne into the container
return ScrapeResponse(**loads(resp.text))
except Exception as e: # @TODO
return None
async def request(method,
url,
timeout,
*,
data=None,
headers=None,
stream=False,
client=None,
**kwargs):
"""Returns a Response object, to be awaited."""
if not client:
client = http3.AsyncClient()
return await client.request(method=method,
url=url,
headers=headers,
stream=stream,
data=data,
**kwargs)
async def get_transaction(client_url: str, tx_id: str) -> Transaction:
"""Get transaction by hash
:param client_url: The haskoin API url
:type client_url: str
:param tx_id: The transaction id
:type tx_id: str
:returns: Transaction info
:raises: 'failed to query transaction by a given hash' if failed to query transaction by a given hash
"""
try:
api_url = f'{client_url}/transaction/{tx_id}'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
balance_response = json.loads(response.content.decode('utf-8'))
inputs = []
for i in balance_response['inputs']:
inputs.append(
TransactionInput(i["pkscript"], i['value'], i['address'],
i['witness'], i['sequence'], i['output'],
i['sigscript'], i['coinbase'], i['txid']))
outputs = []
for i in balance_response['outputs']:
outputs.append(
TransactionOutput(i["spent"], i['pkscript'], i['value'],
i['address'], i['spender']))
result = Transaction(
balance_response["time"], balance_response["size"], inputs,
balance_response["weight"], balance_response["fee"],
balance_response["locktime"], balance_response["block"],
outputs, balance_response["version"],
balance_response["deleted"], balance_response["rbf"],
balance_response["txid"])
return result
else:
raise Exception('failed to query transaction by a given hash')
except Exception as err:
raise Exception(str(err))
async def get_suggested_tx_fee():
"""Get Bitcoin suggested transaction fee
Note: sochain does not provide fee rate related data
This number is from https://bitcoinfees.earn.com API
Refer: https://bitcoinfees.earn.com/api
:returns: The Bitcoin suggested transaction fee per bytes in sat
"""
try:
api_url = 'https://bitcoinfees.earn.com/api/v1/fees/recommended'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
return json.loads(response.content.decode('utf-8'))['fastestFee']
else:
return None
except Exception as err:
raise Exception(str(err))
async def stress_test(requests, size):
geoms = [get_random_box(size) for i in range(0, requests)]
for geom in geoms:
print(geom.wkt)
response_futures = []
async with http3.AsyncClient(timeout=60) as client:
for geom in geoms:
payload = {
"geometry":
mapping(geom),
"geostore_origin":
"rw",
"group_by": ["umd_tree_cover_loss__year"],
"filters": [
"is__umd_regional_primary_forest_2001",
"umd_tree_cover_density_2000__30",
],
"sum": ["area__ha"],
}
print("Sending request...")
response_futures.append(client.post(API_URI, json=payload))
responses = await asyncio.gather(*response_futures)
for response in responses:
if response.status_code != 200:
print("ERROR")
print(geom)
print(response.content)
response = response.json()
if response["status"] != "success":
print("ERROR")
print(geom)
print(response)
print("SUCCESS")
async def get_unspent_txs(sochain_url, network, address):
"""Get address balance
https://sochain.com/api#get-unspent-tx
:param network: testnet or mainnet
:type network: str
:param address: address
:type address: str
:returns: A list of utxo's
"""
try:
api_url = f'{sochain_url}/get_tx_unspent/{to_sochain_network(network)}/{address}'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
txs = json.loads(response.content.decode('utf-8'))['data']['txs']
return txs
except Exception as err:
raise Exception(str(err))
async def get_transactions(sochain_url: str, net: str, address: str):
"""Get address information
https://sochain.com/api#get-display-data-address
:param net: mainnet or testnet
:type net: str
:param address: wallet address
:type address: str
:returns: The fees with memo
"""
try:
api_url = f'{sochain_url}/address/{to_sochain_network(net)}/{address}'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
return json.loads(response.content.decode('utf-8'))['data']
else:
return None
except Exception as err:
raise Exception(str(err))
async def get_tx(sochain_url: str, net: str, hash: str):
"""Get transaction by hash
https://sochain.com/api#get-tx
:param net: mainnet or testnet
:type net: str
:param hash: The transaction hash
:type hash: str
:returns: The fees with memo
"""
try:
api_url = f'{sochain_url}/get_tx/{to_sochain_network(net)}/{hash}'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
return json.loads(response.content.decode('utf-8'))['data']
else:
return None
except Exception as err:
raise Exception(str(err))
async def get_suggested_tx_fee():
"""Get Litecoin suggested transaction fee
Note: sochain does not provide fee rate related data
So use Bitgo API for fee estimation
Refer: https://app.bitgo.com/docs/#operation/v2.tx.getfeeestimate
:returns: The Litecoin suggested transaction fee per bytes in sat
"""
try:
api_url = 'https://app.bitgo.com/api/v2/ltc/tx/fee'
client = http3.AsyncClient()
response = await client.get(api_url)
if response.status_code == 200:
return json.loads(response.content.decode(
'utf-8'))['feePerKb'] / 1000 # feePerKb to feePerByte
else:
return None
except Exception as err:
raise Exception(str(err))
