def test_to_error(): res = hrequest("https://httpbin222.org/ip") assert res.to_error().data.http_code == res.http_code assert res.to_error().is_error() assert res.to_error().error.startswith("connection_error") assert res.is_connection_error() res = hrequest("https://httpbin222.org/ip") assert res.to_error("bla").error == "bla"
def test_post_method(): res = hrequest("https://httpbin.org/post", method="post", params={"a": 1}, json_params=True) assert res.json["data"] == '{"a": 1}' res = hrequest("https://httpbin.org/post", method="POST", params={"a": 1}, json_params=True) assert res.json["data"] == '{"a": 1}'
def get_top_holders(token_address: str, timeout=10, proxy=None) -> Result[list[str]]: url = f"https://etherscan.io/token/generic-tokenholders2?a={token_address}" res = hrequest(url, user_agent=FIREFOX_USER_AGENT, timeout=timeout, proxy=proxy) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: soap = BeautifulSoup(res.body, "html.parser") result = [] for a in soap.select("#maintable table td a"): m = re.search(r"\?a=(.+)$", a["href"]) if m: result.append(m.group(1)) if result: return res.to_ok(result) else: div = soap.select_one("#maintable table div.alert-warning") if "there are no matching entries" in div.text.lower(): return res.to_ok([]) else: return res.to_error("html_parse_error") except Exception as e: return res.to_error(f"exception: {str(e)}")
def get_token_usd_price(address: str, timeout=10, proxy=None) -> Result[Decimal]: url = f"https://etherscan.io/token/{address}" res = hrequest(url, proxy=proxy, user_agent=FIREFOX_USER_AGENT, timeout=timeout) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: soap = BeautifulSoup(res.body, "html.parser") span = soap.select_one( "#ContentPlaceHolder1_tr_valuepertoken span.d-block") if span and "@" in span.text: return res.to_ok( Decimal(span.text.split("@")[0].strip().replace("$", ""))) # check if it's a ERC721 token span = soap.select_one( "#ContentPlaceHolder1_divSummary .card-header-title .text-secondary.small" ) if span and "ERC-721" in span.text: return res.to_error("erc721") return res.to_error("html_parse_error") except Exception as e: return res.to_error(f"exception: {str(e)}")
def get_contract_source_code(api_key: str, address: str, timeout=10, proxy=None) -> Result[EtherscanSourceCode]: params = { "module": "contract", "action": "getsourcecode", "address": address, "apikey": api_key } res = hrequest("https://api.etherscan.io/api", params=params, proxy=proxy, timeout=timeout) try: if res.is_error(): return res.to_error() contract_name = res.json["result"][0]["ContractName"] abi = res.json["result"][0]["ABI"] source_code = res.json["result"][0]["SourceCode"] if "source code not verified" in abi: return res.to_error("not_verified") elif contract_name: abi = json.loads(abi) return res.to_ok( EtherscanSourceCode(source_code=source_code, contract_name=contract_name, abi=abi)) else: return res.to_error("unknown_response") except Exception as e: return res.to_error(f"exception: {str(e)}")
def api_redirect( request: Request, url: str, method: str, data: Optional[str] = None, api_key: APIKey = Depends(self._get_api_key()), ): method = method.lower() headers = {self.api_key_name: api_key} url = str(request.base_url).removesuffix("/") + url if self.app.app_config.use_https: url = url.replace("http://", "https://", 1) params = None if data: params = json.loads(data) res = hrequest(url, method=method, headers=headers, params=params, json_params=True, timeout=600) if res.json: return res.json return res.body
def test_get_params(): res = hrequest("https://httpbin.org/get", params={ "a": 123, "b": "bla bla" }) assert res.json["args"]["b"] == "bla bla"
def get_account_txs( api_key: str, address: str, from_block=None, to_block=None, sort=None, page=None, offset=None, timeout=10, proxy=None, ) -> Result[list[EtherscanTx]]: params = { "module": "account", "action": "txlist", "address": address, "apikey": api_key, } if from_block is not None: params["startblock"] = from_block if to_block is not None: params["endblock"] = to_block if sort: params["sort"] = sort if page is not None: params["page"] = page if offset is not None: params["offset"] = offset res = hrequest("https://api.etherscan.io/api", params=params, proxy=proxy, timeout=timeout) if res.is_error(): return res.to_error() try: if res.json.get("message") == "OK": result = [] for tx in res.json["result"]: result.append( EtherscanTx( block_number=int(tx["blockNumber"]), timestamp=int(tx["timeStamp"]), hash=tx["hash"], transaction_index=int(tx["transactionIndex"]), nonce=int(tx["nonce"]), from_address=tx["from"], to_address=tx["to"], value=int(tx["value"]), gas=int(tx["gas"]), gas_price=int(tx["gasPrice"]), is_error=tx["isError"] == "1", gas_used=int(tx["gasUsed"]), input=tx["input"], ), ) return res.to_ok(result) else: return res.to_error("unknown_response") except Exception as e: return res.to_error(f"exception: {str(e)}")
def test_proxy(): load_dotenv() proxy_url = os.getenv("PROXY", "") proxy = urlparse(proxy_url) res = hrequest("https://httpbin.org/ip", proxy=proxy_url) assert proxy.hostname in res.json["origin"]
def get_address_info(address: str, timeout=10, proxy=None) -> Result[EtherscanInfo]: url = "https://etherscan.io/address/" + address res = hrequest(url, proxy=proxy, user_agent=FIREFOX_USER_AGENT, timeout=timeout) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: soap = BeautifulSoup(res.body, "html.parser") if soap.select_one("#mainaddress"): name = "" tags = [] site = "" tokens_usd = Decimal() # name name_el = soap.select_one( "span[title='Public Name Tag (viewable by anyone)']") if name_el: name = name_el.text.strip() # tags for link in soap.select("#content a.u-label"): if link.get("href") and link["href"].startswith( "/accounts/label/"): tags.append(link["href"].replace("/accounts/label/", "")) # token_usd tokens_usd_el = soap.select_one( "#ContentPlaceHolder1_tokenbalance #availableBalanceDropdown") if tokens_usd_el: m = re.search(r"\$([\d.,]+)", tokens_usd_el.text) if m: value = m.group(1).replace(",", "").replace("...", "").strip() tokens_usd = Decimal(value) # site site_el = soap.select_one("a[title='External Site - More Info']") if site_el: site = site_el.get("href", "") token = None if soap.select_one("a[title='View Token Tracker Page']"): token = get_etherscan_token(address, timeout, proxy).ok return res.to_ok( EtherscanInfo(**md(name, tags, site, tokens_usd, token))) return res.to_error("unknown_response") except Exception as e: return res.to_error(f"exception: {str(e)}")
def get_parity_trace(tx_hash: str, timeout=10, proxy=None) -> Result[ParityTrace]: url = f"https://etherscan.io/vmtrace?txhash={tx_hash}&type=parity#raw" res = hrequest(url, timeout=timeout, proxy=proxy, user_agent=FIREFOX_USER_AGENT) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: soap = BeautifulSoup(res.body, "html.parser") editor_el = soap.select_one("#editor") if not editor_el: return res.to_error("unknown_response") traces_json = json.loads(f"[{editor_el.text}]") block_number = traces_json[0]["blockNumber"] transaction_position = traces_json[0]["transactionPosition"] traces: list[ParityTrace.Trace] = [] for t in traces_json: call_type = t["action"]["callType"] from_ = t["action"]["from"] to = t["action"].get("to") gas = int(t["action"]["gas"], 16) value = int(t["action"]["value"], 16) input_ = t["action"]["input"] gas_used = int(t["result"]["gasUsed"], 16) output = t["result"]["output"] trace = md(call_type, from_, to, gas, value, gas_used, output, input=input_) traces.append(ParityTrace.Trace(**trace)) parity_trace = ParityTrace(block_number=block_number, transaction_position=transaction_position, traces=traces) return res.to_ok(parity_trace) except Exception as e: return res.to_error(f"exception: {str(e)}")
def get_top_accounts(timeout=10, proxy=None) -> Result[list[str]]: url = "https://etherscan.io/accounts" res = hrequest(url, user_agent=FIREFOX_USER_AGENT, timeout=timeout, proxy=proxy) if res.is_error(): return res.to_error() try: soap = BeautifulSoup(res.body, "html.parser") result = [] for a in soap.select("#content table td a"): link = a.get("href", "") if link.startswith("/address/0x"): result.append(link.replace("/address/", "")) return res.to_ok(result) except Exception as e: return res.to_error(f"exception: {str(e)}")
def get_token_transfers_count(address: str, timeout=10, proxy=None) -> Result[int]: url = f"https://etherscan.io/token/generic-tokentxns2?m=normal&contractAddress={address}&a=" res = hrequest(url, timeout=timeout, proxy=proxy, user_agent=FIREFOX_USER_AGENT) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: m = re.search(r"total of ([\d,]+) transaction", res.body) if m: return res.to_ok(int(m.group(1).replace(",", ""))) else: return res.to_error("unknown_response") except Exception as e: return res.to_error(f"exception: {str(e)}")
def get_contract_creation_tx_hash(contract_address: str, timeout=10, proxy=None) -> Result[str]: url = f"https://etherscan.io/address/{contract_address}" res = hrequest(url, timeout=timeout, proxy=proxy, user_agent=FIREFOX_USER_AGENT) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: soap = BeautifulSoup(res.body, "html.parser") a = soap.select_one("a.hash-tag[title='Creator Txn Hash']") if a and a.text and a.text.startswith("0x"): return res.to_ok(a.text) else: return res.to_error("not_found") except Exception as e: return res.to_error(f"exception: {str(e)}")
def _http_call(node: str, data: dict, timeout: int, proxy: Optional[str]) -> Result: res = hrequest(node, method="POST", proxy=proxy, timeout=timeout, params=data, json_params=True) try: if res.is_error(): return res.to_error() err = res.json.get("error", {}).get("message", "") if err: return res.to_error(f"service_error: {err}") if "result" in res.json: return res.to_ok(res.json["result"]) return res.to_error("unknown_response") except Exception as e: return res.to_error(f"exception: {str(e)}")
def work(self, pk): self.log.debug("work(%s)", pk) worker = self.db.worker.get_or_none(pk) if not worker or not worker.started: return False r = hrequest(worker.source, timeout=self.system_service.get_bot().timeout) worker_updated = {"last_work_at": utc_now()} data: Dict[str, Any] = {"worker": worker.name} if r.is_timeout_error(): data["status"] = DataStatus.timeout elif not r.is_error(): if r.json_parse_error: data["status"] = DataStatus.json_error else: data["status"] = DataStatus.ok data["data"] = r.json else: data["status"] = DataStatus.error self.db.data.insert_one(Data(**data)) self.db.worker.update_by_id(pk, {"$set": worker_updated}) return True
def test_json_parse_error(): res = hrequest("https://httpbin.org") assert res.json_parse_error
def test_proxy_error(): res = hrequest("https://httpbin.org/ip", proxy="https://no-real-domain.org:8888") assert res.error == "proxy_error" assert res.is_proxy_error()
def test_timeout(): res = hrequest("https://httpbin.org/delay/10", timeout=2) assert res.error == "timeout" assert res.is_timeout_error()
def get_first_activity(api_key: str, address: str, timeout=10, proxy=None) -> Result[datetime]: oldest_block_time, oldest_block_number = None, None # txlist params = { "module": "account", "action": "txlist", "address": address, "sort": "asc", "page": 1, "offset": 1, "apikey": api_key, } res = hrequest("https://api.etherscan.io/api", params=params, proxy=proxy, timeout=timeout) try: if res.is_error(): return res.to_error() if res.json.get("status") == "1": oldest_block_time = datetime.fromtimestamp( int(pydash.get(res.json, "result[0].timeStamp"))) oldest_block_number = int( pydash.get(res.json, "result[0].blockNumber")) elif "no transactions found" in res.json.get("message", "").lower(): pass else: return res.to_error("unknown_response") # tokentx params = { "module": "account", "action": "tokentx", "address": address, "sort": "asc", "page": 1, "offset": 1, "apikey": api_key, } if oldest_block_number: params["startblock"] = 0 params["endblock"] = oldest_block_number res = hrequest("https://api.etherscan.io/api", params=params, proxy=proxy, timeout=timeout) if res.is_error(): return res.to_error() if res.json.get("status") == "1": oldest_block_time = datetime.fromtimestamp( int(pydash.get(res.json, "result[0].timeStamp"))) elif "no transactions found" in res.json.get("message", "").lower(): pass else: return res.to_error("unknown_response") # txlistinternal don't analyze because of "Query Timeout occured. Please select a smaller result dataset" if oldest_block_time: oldest_block_time = oldest_block_time.replace(tzinfo=timezone.utc) return res.to_ok(oldest_block_time) except Exception as e: return res.to_error(f"exception: {str(e)}")
def task1() -> str: res = hrequest("https://httpbin.org/user-agent", user_agent="agent007") return res.json["user-agent"]
def test_connection_error(): res = hrequest("https://httpbin222.org/ip", timeout=2) assert res.error.startswith("connection_error")
def test_to_ok(): res = hrequest("https://httpbin.org/ip") assert res.to_ok(res.json).data.http_code == res.http_code assert res.to_ok(res.json).is_ok() assert res.to_ok(res.json).ok == res.json
def test_custom_user_agent(): user_agent = "moon cat" res = hrequest("https://httpbin.org/user-agent", user_agent=user_agent) assert user_agent in res.body
def test_firefox_user_agent(): res = hrequest("https://httpbin.org/user-agent", user_agent=FIREFOX_USER_AGENT) assert FIREFOX_USER_AGENT in res.body
def get_etherscan_token(address: str, timeout=10, proxy=None) -> Result[EtherscanToken]: url = f"https://etherscan.io/token/{address}" res = hrequest(url, timeout=timeout, proxy=proxy, user_agent=FIREFOX_USER_AGENT) if res.is_error(): return res.to_error() if "maximum limit reached for this request" in res.body: return res.to_error("request_throttled") try: soap = BeautifulSoup(res.body, "html.parser") if "[ERC-20]" in res.body: erc = "erc20" elif "[ERC-721]" in res.body: erc = "erc721" else: erc = "unknown" symbol = None el = soap.select_one("#ContentPlaceHolder1_hdnSymbol") if el: symbol = el.get("value") total_supply = None el = soap.select_one("#ContentPlaceHolder1_hdnTotalSupply") if el: total_supply = el.get("value") if total_supply: total_supply = Decimal(total_supply.replace(",", "").strip()) decimals = None el = soap.select_one("#ContentPlaceHolder1_trDecimals") if el: decimals = el.text if decimals: decimals = int(decimals.replace("Decimals:", "").strip()) holders_count = None el = soap.select_one("#ContentPlaceHolder1_tr_tokenHolders") if el: holders_count = el.text if holders_count: holders_count = int( holders_count.replace("Holders:", "").replace( "addresses", "").replace("address", "").replace(",", "").strip(), ) price_usd = None el = soap.select_one( "#ContentPlaceHolder1_tr_valuepertoken span.d-block") if el and "@" in el.text: price_usd = Decimal(el.text.split("@")[0].strip().replace("$", "")) market_cap_usd = None el = soap.select_one("#pricebutton") if el and "$" in el.text: market_cap_usd = Decimal(el.text.strip().replace("$", "").replace( ",", "")) site = None el = soap.select_one("#ContentPlaceHolder1_tr_officialsite_1 a") if el: site = el["href"] transfers_count = get_token_transfers_count(address, timeout, proxy).ok info = md(erc, symbol, total_supply, decimals, holders_count, transfers_count, price_usd, market_cap_usd, site) return res.to_ok(EtherscanToken(**info)) except Exception as e: return res.to_error(f"exception: {str(e)}")