def test_to_error():
res = hrequest("https://httpbin222.org/ip")
assert res.to_error().data.http_code == res.http_code
assert res.to_error().is_error()
assert res.to_error().error.startswith("connection_error")
assert res.is_connection_error()
res = hrequest("https://httpbin222.org/ip")
assert res.to_error("bla").error == "bla"
def test_post_method():
res = hrequest("https://httpbin.org/post",
method="post",
params={"a": 1},
json_params=True)
assert res.json["data"] == '{"a": 1}'
res = hrequest("https://httpbin.org/post",
method="POST",
params={"a": 1},
json_params=True)
assert res.json["data"] == '{"a": 1}'
def get_top_holders(token_address: str,
timeout=10,
proxy=None) -> Result[list[str]]:
url = f"https://etherscan.io/token/generic-tokenholders2?a={token_address}"
res = hrequest(url,
user_agent=FIREFOX_USER_AGENT,
timeout=timeout,
proxy=proxy)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
soap = BeautifulSoup(res.body, "html.parser")
result = []
for a in soap.select("#maintable table td a"):
m = re.search(r"\?a=(.+)$", a["href"])
if m:
result.append(m.group(1))
if result:
return res.to_ok(result)
else:
div = soap.select_one("#maintable table div.alert-warning")
if "there are no matching entries" in div.text.lower():
return res.to_ok([])
else:
return res.to_error("html_parse_error")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def get_token_usd_price(address: str,
timeout=10,
proxy=None) -> Result[Decimal]:
url = f"https://etherscan.io/token/{address}"
res = hrequest(url,
proxy=proxy,
user_agent=FIREFOX_USER_AGENT,
timeout=timeout)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
soap = BeautifulSoup(res.body, "html.parser")
span = soap.select_one(
"#ContentPlaceHolder1_tr_valuepertoken span.d-block")
if span and "@" in span.text:
return res.to_ok(
Decimal(span.text.split("@")[0].strip().replace("$", "")))
# check if it's a ERC721 token
span = soap.select_one(
"#ContentPlaceHolder1_divSummary .card-header-title .text-secondary.small"
)
if span and "ERC-721" in span.text:
return res.to_error("erc721")
return res.to_error("html_parse_error")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def get_contract_source_code(api_key: str,
address: str,
timeout=10,
proxy=None) -> Result[EtherscanSourceCode]:
params = {
"module": "contract",
"action": "getsourcecode",
"address": address,
"apikey": api_key
}
res = hrequest("https://api.etherscan.io/api",
params=params,
proxy=proxy,
timeout=timeout)
try:
if res.is_error():
return res.to_error()
contract_name = res.json["result"][0]["ContractName"]
abi = res.json["result"][0]["ABI"]
source_code = res.json["result"][0]["SourceCode"]
if "source code not verified" in abi:
return res.to_error("not_verified")
elif contract_name:
abi = json.loads(abi)
return res.to_ok(
EtherscanSourceCode(source_code=source_code,
contract_name=contract_name,
abi=abi))
else:
return res.to_error("unknown_response")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def api_redirect(
request: Request,
url: str,
method: str,
data: Optional[str] = None,
api_key: APIKey = Depends(self._get_api_key()),
):
method = method.lower()
headers = {self.api_key_name: api_key}
url = str(request.base_url).removesuffix("/") + url
if self.app.app_config.use_https:
url = url.replace("http://", "https://", 1)
params = None
if data:
params = json.loads(data)
res = hrequest(url,
method=method,
headers=headers,
params=params,
json_params=True,
timeout=600)
if res.json:
return res.json
return res.body
def test_get_params():
res = hrequest("https://httpbin.org/get",
params={
"a": 123,
"b": "bla bla"
})
assert res.json["args"]["b"] == "bla bla"
def get_account_txs(
api_key: str,
address: str,
from_block=None,
to_block=None,
sort=None,
page=None,
offset=None,
timeout=10,
proxy=None,
) -> Result[list[EtherscanTx]]:
params = {
"module": "account",
"action": "txlist",
"address": address,
"apikey": api_key,
}
if from_block is not None:
params["startblock"] = from_block
if to_block is not None:
params["endblock"] = to_block
if sort:
params["sort"] = sort
if page is not None:
params["page"] = page
if offset is not None:
params["offset"] = offset
res = hrequest("https://api.etherscan.io/api",
params=params,
proxy=proxy,
timeout=timeout)
if res.is_error():
return res.to_error()
try:
if res.json.get("message") == "OK":
result = []
for tx in res.json["result"]:
result.append(
EtherscanTx(
block_number=int(tx["blockNumber"]),
timestamp=int(tx["timeStamp"]),
hash=tx["hash"],
transaction_index=int(tx["transactionIndex"]),
nonce=int(tx["nonce"]),
from_address=tx["from"],
to_address=tx["to"],
value=int(tx["value"]),
gas=int(tx["gas"]),
gas_price=int(tx["gasPrice"]),
is_error=tx["isError"] == "1",
gas_used=int(tx["gasUsed"]),
input=tx["input"],
), )
return res.to_ok(result)
else:
return res.to_error("unknown_response")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def test_proxy():
load_dotenv()
proxy_url = os.getenv("PROXY", "")
proxy = urlparse(proxy_url)
res = hrequest("https://httpbin.org/ip", proxy=proxy_url)
assert proxy.hostname in res.json["origin"]
def get_address_info(address: str,
timeout=10,
proxy=None) -> Result[EtherscanInfo]:
url = "https://etherscan.io/address/" + address
res = hrequest(url,
proxy=proxy,
user_agent=FIREFOX_USER_AGENT,
timeout=timeout)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
soap = BeautifulSoup(res.body, "html.parser")
if soap.select_one("#mainaddress"):
name = ""
tags = []
site = ""
tokens_usd = Decimal()
# name
name_el = soap.select_one(
"span[title='Public Name Tag (viewable by anyone)']")
if name_el:
name = name_el.text.strip()
# tags
for link in soap.select("#content a.u-label"):
if link.get("href") and link["href"].startswith(
"/accounts/label/"):
tags.append(link["href"].replace("/accounts/label/", ""))
# token_usd
tokens_usd_el = soap.select_one(
"#ContentPlaceHolder1_tokenbalance #availableBalanceDropdown")
if tokens_usd_el:
m = re.search(r"\$([\d.,]+)", tokens_usd_el.text)
if m:
value = m.group(1).replace(",", "").replace("...",
"").strip()
tokens_usd = Decimal(value)
# site
site_el = soap.select_one("a[title='External Site - More Info']")
if site_el:
site = site_el.get("href", "")
token = None
if soap.select_one("a[title='View Token Tracker Page']"):
token = get_etherscan_token(address, timeout, proxy).ok
return res.to_ok(
EtherscanInfo(**md(name, tags, site, tokens_usd, token)))
return res.to_error("unknown_response")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def get_parity_trace(tx_hash: str,
timeout=10,
proxy=None) -> Result[ParityTrace]:
url = f"https://etherscan.io/vmtrace?txhash={tx_hash}&type=parity#raw"
res = hrequest(url,
timeout=timeout,
proxy=proxy,
user_agent=FIREFOX_USER_AGENT)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
soap = BeautifulSoup(res.body, "html.parser")
editor_el = soap.select_one("#editor")
if not editor_el:
return res.to_error("unknown_response")
traces_json = json.loads(f"[{editor_el.text}]")
block_number = traces_json[0]["blockNumber"]
transaction_position = traces_json[0]["transactionPosition"]
traces: list[ParityTrace.Trace] = []
for t in traces_json:
call_type = t["action"]["callType"]
from_ = t["action"]["from"]
to = t["action"].get("to")
gas = int(t["action"]["gas"], 16)
value = int(t["action"]["value"], 16)
input_ = t["action"]["input"]
gas_used = int(t["result"]["gasUsed"], 16)
output = t["result"]["output"]
trace = md(call_type,
from_,
to,
gas,
value,
gas_used,
output,
input=input_)
traces.append(ParityTrace.Trace(**trace))
parity_trace = ParityTrace(block_number=block_number,
transaction_position=transaction_position,
traces=traces)
return res.to_ok(parity_trace)
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def get_top_accounts(timeout=10, proxy=None) -> Result[list[str]]:
url = "https://etherscan.io/accounts"
res = hrequest(url,
user_agent=FIREFOX_USER_AGENT,
timeout=timeout,
proxy=proxy)
if res.is_error():
return res.to_error()
try:
soap = BeautifulSoup(res.body, "html.parser")
result = []
for a in soap.select("#content table td a"):
link = a.get("href", "")
if link.startswith("/address/0x"):
result.append(link.replace("/address/", ""))
return res.to_ok(result)
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def get_token_transfers_count(address: str,
timeout=10,
proxy=None) -> Result[int]:
url = f"https://etherscan.io/token/generic-tokentxns2?m=normal&contractAddress={address}&a="
res = hrequest(url,
timeout=timeout,
proxy=proxy,
user_agent=FIREFOX_USER_AGENT)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
m = re.search(r"total of ([\d,]+) transaction", res.body)
if m:
return res.to_ok(int(m.group(1).replace(",", "")))
else:
return res.to_error("unknown_response")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def get_contract_creation_tx_hash(contract_address: str,
timeout=10,
proxy=None) -> Result[str]:
url = f"https://etherscan.io/address/{contract_address}"
res = hrequest(url,
timeout=timeout,
proxy=proxy,
user_agent=FIREFOX_USER_AGENT)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
soap = BeautifulSoup(res.body, "html.parser")
a = soap.select_one("a.hash-tag[title='Creator Txn Hash']")
if a and a.text and a.text.startswith("0x"):
return res.to_ok(a.text)
else:
return res.to_error("not_found")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def _http_call(node: str, data: dict, timeout: int,
proxy: Optional[str]) -> Result:
res = hrequest(node,
method="POST",
proxy=proxy,
timeout=timeout,
params=data,
json_params=True)
try:
if res.is_error():
return res.to_error()
err = res.json.get("error", {}).get("message", "")
if err:
return res.to_error(f"service_error: {err}")
if "result" in res.json:
return res.to_ok(res.json["result"])
return res.to_error("unknown_response")
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def work(self, pk):
self.log.debug("work(%s)", pk)
worker = self.db.worker.get_or_none(pk)
if not worker or not worker.started:
return False
r = hrequest(worker.source,
timeout=self.system_service.get_bot().timeout)
worker_updated = {"last_work_at": utc_now()}
data: Dict[str, Any] = {"worker": worker.name}
if r.is_timeout_error():
data["status"] = DataStatus.timeout
elif not r.is_error():
if r.json_parse_error:
data["status"] = DataStatus.json_error
else:
data["status"] = DataStatus.ok
data["data"] = r.json
else:
data["status"] = DataStatus.error
self.db.data.insert_one(Data(**data))
self.db.worker.update_by_id(pk, {"$set": worker_updated})
return True
def test_json_parse_error():
res = hrequest("https://httpbin.org")
assert res.json_parse_error
def test_proxy_error():
res = hrequest("https://httpbin.org/ip",
proxy="https://no-real-domain.org:8888")
assert res.error == "proxy_error"
assert res.is_proxy_error()
def test_timeout():
res = hrequest("https://httpbin.org/delay/10", timeout=2)
assert res.error == "timeout"
assert res.is_timeout_error()
def get_first_activity(api_key: str,
address: str,
timeout=10,
proxy=None) -> Result[datetime]:
oldest_block_time, oldest_block_number = None, None
# txlist
params = {
"module": "account",
"action": "txlist",
"address": address,
"sort": "asc",
"page": 1,
"offset": 1,
"apikey": api_key,
}
res = hrequest("https://api.etherscan.io/api",
params=params,
proxy=proxy,
timeout=timeout)
try:
if res.is_error():
return res.to_error()
if res.json.get("status") == "1":
oldest_block_time = datetime.fromtimestamp(
int(pydash.get(res.json, "result[0].timeStamp")))
oldest_block_number = int(
pydash.get(res.json, "result[0].blockNumber"))
elif "no transactions found" in res.json.get("message", "").lower():
pass
else:
return res.to_error("unknown_response")
# tokentx
params = {
"module": "account",
"action": "tokentx",
"address": address,
"sort": "asc",
"page": 1,
"offset": 1,
"apikey": api_key,
}
if oldest_block_number:
params["startblock"] = 0
params["endblock"] = oldest_block_number
res = hrequest("https://api.etherscan.io/api",
params=params,
proxy=proxy,
timeout=timeout)
if res.is_error():
return res.to_error()
if res.json.get("status") == "1":
oldest_block_time = datetime.fromtimestamp(
int(pydash.get(res.json, "result[0].timeStamp")))
elif "no transactions found" in res.json.get("message", "").lower():
pass
else:
return res.to_error("unknown_response")
# txlistinternal don't analyze because of "Query Timeout occured. Please select a smaller result dataset"
if oldest_block_time:
oldest_block_time = oldest_block_time.replace(tzinfo=timezone.utc)
return res.to_ok(oldest_block_time)
except Exception as e:
return res.to_error(f"exception: {str(e)}")
def task1() -> str:
res = hrequest("https://httpbin.org/user-agent", user_agent="agent007")
return res.json["user-agent"]
def test_connection_error():
res = hrequest("https://httpbin222.org/ip", timeout=2)
assert res.error.startswith("connection_error")
def test_to_ok():
res = hrequest("https://httpbin.org/ip")
assert res.to_ok(res.json).data.http_code == res.http_code
assert res.to_ok(res.json).is_ok()
assert res.to_ok(res.json).ok == res.json
def test_custom_user_agent():
user_agent = "moon cat"
res = hrequest("https://httpbin.org/user-agent", user_agent=user_agent)
assert user_agent in res.body
def test_firefox_user_agent():
res = hrequest("https://httpbin.org/user-agent",
user_agent=FIREFOX_USER_AGENT)
assert FIREFOX_USER_AGENT in res.body
def get_etherscan_token(address: str,
timeout=10,
proxy=None) -> Result[EtherscanToken]:
url = f"https://etherscan.io/token/{address}"
res = hrequest(url,
timeout=timeout,
proxy=proxy,
user_agent=FIREFOX_USER_AGENT)
if res.is_error():
return res.to_error()
if "maximum limit reached for this request" in res.body:
return res.to_error("request_throttled")
try:
soap = BeautifulSoup(res.body, "html.parser")
if "[ERC-20]" in res.body:
erc = "erc20"
elif "[ERC-721]" in res.body:
erc = "erc721"
else:
erc = "unknown"
symbol = None
el = soap.select_one("#ContentPlaceHolder1_hdnSymbol")
if el:
symbol = el.get("value")
total_supply = None
el = soap.select_one("#ContentPlaceHolder1_hdnTotalSupply")
if el:
total_supply = el.get("value")
if total_supply:
total_supply = Decimal(total_supply.replace(",", "").strip())
decimals = None
el = soap.select_one("#ContentPlaceHolder1_trDecimals")
if el:
decimals = el.text
if decimals:
decimals = int(decimals.replace("Decimals:", "").strip())
holders_count = None
el = soap.select_one("#ContentPlaceHolder1_tr_tokenHolders")
if el:
holders_count = el.text
if holders_count:
holders_count = int(
holders_count.replace("Holders:", "").replace(
"addresses",
"").replace("address", "").replace(",", "").strip(), )
price_usd = None
el = soap.select_one(
"#ContentPlaceHolder1_tr_valuepertoken span.d-block")
if el and "@" in el.text:
price_usd = Decimal(el.text.split("@")[0].strip().replace("$", ""))
market_cap_usd = None
el = soap.select_one("#pricebutton")
if el and "$" in el.text:
market_cap_usd = Decimal(el.text.strip().replace("$", "").replace(
",", ""))
site = None
el = soap.select_one("#ContentPlaceHolder1_tr_officialsite_1 a")
if el:
site = el["href"]
transfers_count = get_token_transfers_count(address, timeout, proxy).ok
info = md(erc, symbol, total_supply, decimals, holders_count,
transfers_count, price_usd, market_cap_usd, site)
return res.to_ok(EtherscanToken(**info))
except Exception as e:
return res.to_error(f"exception: {str(e)}")