def __init__(
self,
wallet_bin_path,
datastore_path,
wallet_password,
):
"""
Manage a Parity wallet for Ethereum.
The wallet key file must be in `datastore_path/keys/ethereum/`.
Multiple "accounts" (wallet key files) are not supported - see the `accounts` property.
:param wallet_bin_path: Path to Parity wallet executable
:param datastore_path: Path to datastore directory (which includes wallet files and blockchain)
:param wallet_password: Password to enter for decrypting the account
"""
self.wallet_bin_path = wallet_bin_path
self.datastore_path = datastore_path
self.wallet_password = wallet_password
self._server = None
self._accounts = None
self._block_timestamps = {}
self.ec = EtherChain()
class EtherchainAccountTest(unittest.TestCase):
def setUp(self):
self.etherchain = EtherChain()
def test_tx_pending(self):
self.assertIn(
"recordsTotal",
self.etherchain.transactions_pending(start=0, length=1).keys())
def test_txs(self):
self.assertIn("recordsTotal",
self.etherchain.transactions(start=0, length=1).keys())
# yeah super lazy :p
def test_blocks(self):
self.assertIn("recordsTotal",
self.etherchain.blocks(start=0, length=1).keys())
# we'll test content later
def test_accounts(self):
self.assertIn("recordsTotal",
self.etherchain.accounts(start=0, length=1).keys())
def test_contracts(self):
self.assertIn("processed",
self.etherchain.contracts(start=0, length=1).keys())
def __init__(self, proxies={}):
self.config = configparser.ConfigParser()
self.config.read('config.ini')
self.session = UserAgent(baseurl="https://etherscan.io",
retry=5,
retrydelay=8,
proxies=proxies)
self.ec = EtherChain()
self.soup = None
def iter_contracts(start=0, length=100):
s = EtherChain(proxies={})
while True:
import time
time.sleep(30)
s = EtherChain()
contracts = s.contracts(start=start, length=length)
for contract in contracts["data"]:
yield contract
start += contracts["processed"]
def setUp(self):
self.etherchain = EtherChain()
class EtherScanIoApi(object):
"""
Base EtherScan.io Api implementation
TODO:
- implement a script (client) that runs all the python script
- fix the issue about SC with several classes. The issue is at 03 script
- Fix the issue about solmet, for some address the tool is not able to get statistic at 02 and it brokes 03
- fix _get_contract_name
"""
def __init__(self, proxies={}):
self.config = configparser.ConfigParser()
self.config.read('config.ini')
self.session = UserAgent(baseurl="https://etherscan.io",
retry=5,
retrydelay=8,
proxies=proxies)
self.ec = EtherChain()
self.soup = None
def get_contracts_from_block(self, block):
soup = BeautifulSoup(requests.get('https://etherscan.io/txs?block=' +
str(block)).text,
features="html.parser")
addresses = soup.select("i[title='Contract']")
for address in list(
set(
map(
lambda x: x.findNext('a')['href'].replace(
'/address/', ''), addresses))):
if not self._is_new_address(address):
continue
describe_contract = self.ec.account(address).describe_contract
self._set_soup(address)
contract = {
'address': address,
'name': self._get_contract_name(),
'compiler': None,
'compiler_version': self._get_compiler_version(),
'balance': describe_contract.__self__['balance'],
'txcount': describe_contract.__self__['txreceived'],
'firstseen': describe_contract.__self__['firstseen'],
'lastseen': describe_contract.__self__['lastseen']
}
yield contract
def get_contracts_from_etherscan(self, start=0, end=None):
page = start
while not end or page <= end:
resp = self.session.get("/contractsVerified/%d" % page).text
page, lastpage = re.findall(
r'Page <.*>(\d+)</.*> of <.*>(\d+)</.*>', resp)[0]
page, lastpage = int(page), int(lastpage)
if not end:
end = lastpage
rows = self._parse_tbodies(resp)[0] # only use first tbody
for col in rows:
address = self._extract_text_from_html(col[0]).split(" ", 1)[0]
if not self._is_new_address(address):
continue
describe_contract = self.ec.account(address).describe_contract
firstseen = describe_contract.__self__['firstseen']
lastseen = describe_contract.__self__['lastseen']
contract = {
'address':
address,
'name':
self._extract_text_from_html(col[1]),
'compiler':
self._extract_text_from_html(col[2]),
'compiler_version':
self._extract_text_from_html(col[3]),
'balance':
self._get_balance(self._extract_text_from_html(col[4])),
'txcount':
self._extract_text_from_html(col[5]),
'firstseen':
firstseen,
'lastseen':
firstseen
}
yield contract
page += 1
def write_etherChain_fn(self, contracts=[]):
amount = 100
for nr, c in enumerate(contracts):
with open(self.config['DEFAULT']['etherChain_fn'], 'a+') as f:
print("got contract: %s" % c)
f_path = os.path.join(self.config['DEFAULT']['output_path'],
'%s.sol' % (c["address"]))
try:
source = self._get_contract_source(c["address"]).strip()
if not len(source):
raise Exception(c)
except Exception as e:
continue
f.write("%s\n" % c)
with open(f_path, "wb") as f:
f.write(bytes(source, "utf8"))
print("[%d/%d] dumped --> %s (%-20s) -> %s" %
(nr, amount, c["address"], c["name"], f_path))
nr += 1
if nr >= amount:
break
def _get_contract_source(self, address):
import time
e = None
for _ in range(5):
resp = self.session.get("/address/%s" % address).text
print("/address/%s" % address)
if "You have reached your maximum request limit for this resource. Please try again later" in resp:
print("[[THROTTELING]]")
time.sleep(1 + 2.5 * _)
continue
try:
print(
"=======================================================")
print(address)
resp = resp.split(
"<pre class='js-sourcecopyarea editor' id='editor' style='margin-top: 5px;'>",
1)[1]
resp = resp.split("</pre><br>", 1)[0]
return resp.replace("<", "<").replace(">", ">").replace(
"≤",
"<=").replace("≥",
">=").replace("&",
"&").replace("|", "|")
except:
print(traceback.format_exc())
time.sleep(1 + 2.5 * _)
break
def _is_new_address(self, address):
if (address not in open(self.config['DEFAULT']['smec_fn']).read()):
return True
return False
def _set_soup(self, address):
url = address.join(['https://etherscan.io/address/', '#code'])
self.soup = BeautifulSoup(requests.get(url).text, 'html.parser')
def _get_compiler_version(self):
try:
str = self.soup.findAll('span',
text=re.compile('v0.'))[0].contents[0]
return re.search('v(\d{1,2}.\d{1,2}.\d{1,2})', str)[1]
except IndexError:
return None
def _get_contract_name(self):
try:
return self.soup.find(lambda tag: tag.name == "span" and "Name" in
tag.text).parent.find_next(
'td').contents[0].strip()
except:
return None
def _get_addresses_from_fn(self, fn):
try:
fp = open(fn)
return list(filter(None, map(lambda x: x.strip(), fp.readlines())))
finally:
fp.close()
def _extract_text_from_html(self, s):
return re.sub('<[^<]+?>', '', s).strip()
def _extract_hexstr_from_html_attrib(self, s):
return ''.join(re.findall(r".+/([^']+)'",
s)) if ">" in s and "</" in s else s
def _get_balance(self, balance):
try:
return int(re.sub('[a-zA-Z]', '', balance))
except ValueError:
return None
def _get_pageable_data(self, path, start=0, length=10):
params = {
"start": start,
"length": length,
}
resp = self.session.get(path, params=params).json()
# cleanup HTML from response
for item in resp['data']:
keys = item.keys()
for san_k in set(keys).intersection(
set(("account", "blocknumber", "type", "direction"))):
item[san_k] = self._extract_text_from_html(item[san_k])
for san_k in set(keys).intersection(
("parenthash", "from", "to", "address")):
item[san_k] = self._extract_hexstr_from_html_attrib(
item[san_k])
return resp
def _parse_tbodies(self, data):
tbodies = []
for tbody in re.findall(r"<tbody.*?>(.+?)</tbody>", data, re.DOTALL):
rows = []
for tr in re.findall(r"<tr.*?>(.+?)</tr>", tbody):
rows.append(re.findall(r"<td.*?>(.+?)</td>", tr))
tbodies.append(rows)
return tbodies
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pyetherchain.pyetherchain import EtherChain
e = EtherChain()
# getting an accoutn object
ac = e.account("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")
# show the account object (json), retrieve the source if available, show transactions
print(ac)
print(ac.source)
print(ac.swarm_hash)
print(ac.transactions())
print(ac.history())
# access the charts api
print(e.charts.market_cap())
# retrieve hardfork information
print(e.hardforks())
# list pending transaactions (takes arguments)
# print(e.transactions_pending())
# describe the constructor invokation and other transaction in a human readable way
contract = e.account("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")
# print("constructor:{}".format(contract.abi.describe_constructor(contract.constructor_args)))
# for tx in contract.transactions(direction="in", length=10000)["data"]:
# tx_obj = e.transaction(tx["parenthash"])[0]
# print("transaction: [IN] <== %s : %s".format((str(tx_obj["hash"]), str(contract.abi.describe_input(tx_obj["input"])))))
# or just shorthand dump contract with extra info
import pymongo
import datetime
import requests
from pymongo import MongoClient
from HTMLParser import HTMLParser
from pyetherchain.pyetherchain import EtherChain
from pyetherchain.pyetherchain import EtherChainApi
DEBUG_MODE = False
KEYWORDS = [
"scam", "honeypot", "honey-pot", "honey pot", "honey trap", "honey",
"trap", "fraud"
]
etherchain = EtherChain()
etherchain_api = EtherChainApi()
prices = etherchain_api.get_stats_price_usd()
def get_one_eth_to_usd(timestamp):
one_eth_to_usd = prices[-1]["value"]
for index, price in enumerate(prices):
if index < len(prices) - 1:
if prices[index]["time"] <= timestamp and timestamp <= prices[
index + 1]["time"]:
one_eth_to_usd = prices[index]["value"]
break
return one_eth_to_usd
def main():
address = sys.argv[1]
print(EtherChain().account(address).code)
def __init__(self, proxies={}):
self.session = UserAgent(baseurl="https://etherscan.io",
retry=5,
retrydelay=8,
proxies=proxies)
self.ec = EtherChain()
class EtherScanIoApi(object):
"""
Base EtherScan.io Api implementation
"""
def __init__(self, proxies={}):
self.session = UserAgent(baseurl="https://etherscan.io",
retry=5,
retrydelay=8,
proxies=proxies)
self.ec = EtherChain()
def get_contracts(self, start=0, end=None):
page = start
while not end or page <= end:
resp = self.session.get("/contractsVerified/%d" % page).text
page, lastpage = re.findall(
r'Page <.*>(\d+)</.*> of <.*>(\d+)</.*>', resp)[0]
page, lastpage = int(page), int(lastpage)
if not end:
end = lastpage
rows = self._parse_tbodies(resp)[0] # only use first tbody
for col in rows:
address = self._extract_text_from_html(col[0]).split(" ", 1)[0]
describe_contract = self.ec.account(address).describe_contract
firstseen = describe_contract.__self__['firstseen']
lastseen = describe_contract.__self__['lastseen']
contract = {
'address':
address,
'name':
self._extract_text_from_html(col[1]),
'compiler':
self._extract_text_from_html(col[2]),
'compiler_version':
self._extract_text_from_html(col[3]),
'balance':
self._get_balance(self._extract_text_from_html(col[4])),
'txcount':
self._extract_text_from_html(col[5]),
'firstseen':
firstseen,
'lastseen':
firstseen
}
yield contract
page += 1
def get_contract_source(self, address):
import time
e = None
for _ in range(20):
resp = self.session.get("/address/%s" % address).text
if "You have reached your maximum request limit for this resource. Please try again later" in resp:
print("[[THROTTELING]]")
time.sleep(1 + 2.5 * _)
continue
try:
print(
"=======================================================")
print(address)
resp = resp.split(
"</div><pre class='js-sourcecopyarea' id='editor' style='margin-top: 5px;'>",
1)[1]
resp = resp.split("</pre><br>", 1)[0]
return resp.replace("<", "<").replace(">", ">").replace(
"≤",
"<=").replace("≥",
">=").replace("&",
"&").replace("|", "|")
except Exception as e:
print(e)
time.sleep(1 + 2.5 * _)
continue
raise e
def _extract_text_from_html(self, s):
return re.sub('<[^<]+?>', '', s).strip()
def _extract_hexstr_from_html_attrib(self, s):
return ''.join(re.findall(r".+/([^']+)'",
s)) if ">" in s and "</" in s else s
def _get_balance(self, balance):
try:
return int(re.sub('[a-zA-Z]', '', balance))
except ValueError:
return None
def _get_pageable_data(self, path, start=0, length=10):
params = {
"start": start,
"length": length,
}
resp = self.session.get(path, params=params).json()
# cleanup HTML from response
for item in resp['data']:
keys = item.keys()
for san_k in set(keys).intersection(
set(("account", "blocknumber", "type", "direction"))):
item[san_k] = self._extract_text_from_html(item[san_k])
for san_k in set(keys).intersection(
("parenthash", "from", "to", "address")):
item[san_k] = self._extract_hexstr_from_html_attrib(
item[san_k])
return resp
def _parse_tbodies(self, data):
tbodies = []
for tbody in re.findall(r"<tbody.*?>(.+?)</tbody>", data, re.DOTALL):
rows = []
for tr in re.findall(r"<tr.*?>(.+?)</tr>", tbody):
rows.append(re.findall(r"<td.*?>(.+?)</td>", tr))
tbodies.append(rows)
return tbodies
class PiggyETH:
TXN_GAS_LIMIT = 21000
def __init__(
self,
wallet_bin_path,
datastore_path,
wallet_password,
):
"""
Manage a Parity wallet for Ethereum.
The wallet key file must be in `datastore_path/keys/ethereum/`.
Multiple "accounts" (wallet key files) are not supported - see the `accounts` property.
:param wallet_bin_path: Path to Parity wallet executable
:param datastore_path: Path to datastore directory (which includes wallet files and blockchain)
:param wallet_password: Password to enter for decrypting the account
"""
self.wallet_bin_path = wallet_bin_path
self.datastore_path = datastore_path
self.wallet_password = wallet_password
self._server = None
self._accounts = None
self._block_timestamps = {}
self.ec = EtherChain()
@property
def server(self):
if self._server is None:
ipc = Web3.IPCProvider(
os.path.join(self.datastore_path, 'jsonrpc.ipc'))
self._server = Web3(ipc)
return self._server
@property
def accounts(self):
if self._accounts is None:
self._accounts = self.server.eth.accounts
# We only handle 1 account (address) as of now.
# In order to support multiple accounts you need to alter:
# `get_receive_address`, `get_balance`, `transactions_since`, `perform_transaction`
assert len(self.accounts) == 1
return self._accounts
def start_server(self):
if self._rpc_loaded():
return
command = [
self.wallet_bin_path,
'-d',
self.datastore_path,
'--log-file',
os.path.join(os.getcwd(), self.datastore_path, 'parity_log.txt'),
'--no-ancient-blocks',
'--no-ws',
'--no-jsonrpc',
'--ipc-apis=web3,eth,personal', # We need personal to actually perform a transaction
'--warp-barrier',
'5842205',
]
logger.info("Starting ETH wallet: {}".format(' '.join(command)))
popen_spawn.PopenSpawn(command)
wait_for_success(self._rpc_loaded, 'ETH RPC')
def stop_server(self):
"""Use pkill to kill the parity wallet."""
p = pexpect.spawn('/usr/bin/pkill', ['-f', self.wallet_bin_path])
p.wait()
if p.status is not 0:
raise ValueError('Error pkilling ETH:\n{}'.format(p.read()))
def _rpc_loaded(self):
"""Attempt a connection to the RPC"""
try:
self.server.eth.getBlock(self.server.eth.blockNumber)
return True
except (web3.utils.threads.Timeout, ConnectionRefusedError,
FileNotFoundError):
return False
def _from_wei(self, wei):
"""Convert wei to ether"""
return self.server.fromWei(wei, 'ether')
def get_receive_address(self):
return self.accounts[0]
def get_balance(self):
wei = self.server.eth.getBalance(self.get_receive_address(), 'latest')
return self._from_wei(wei)
def suggest_miner_fee(self):
gas_needed = self.server.eth.estimateGas({
'from':
self.get_receive_address(),
'to':
self.get_receive_address(),
'value':
1
})
# Get the gas price in the latest unconfirmed block
gas_price = self.server.eth.gasPrice
return self._from_wei(gas_price * gas_needed)
def transactions_since(self, since_unix_time, only_look_at=10):
"""
Gets transactions since specified unix timestamp.
We use Etherchain.org's server, because it's impractical to create an index for ETH
transactions ourselves.
Caveats:
- this has privacy implications (you reveal your ETH address to Etherchain)
- only looks at last `only_look_at` transactions to avoid abusing the server
- only shows 5 decimals after the point (minimum value 0.00001 ETH)
"""
acc = self.ec.account(self.accounts[0])
raw_txns = acc.transactions(length=only_look_at, direction='in')
return self._process_history(raw_txns, since_unix_time,
self._timestamp_getter)
@classmethod
def _process_history(cls, raw_txns, since_unix_time, timestamp_getter):
def to_eth(value_string):
value, unit = value_string.split()
if unit != 'ETH':
raise ValueError(
'Txn not valued in ETH: {}'.format(value_string))
return Decimal(value)
txn_list = [
{
'txid': tx['parenthash'],
'time': timestamp_getter(int(tx['blocknumber'])),
'value': to_eth(tx['value'])
} for tx in raw_txns['data']
if tx['direction'].lower() == 'in' and tx['value'] != '0 ETH'
]
return [
tx for tx in txn_list
if tx['value'] > 0 and tx['time'] >= since_unix_time
]
def _timestamp_getter(self, blocknumber):
if blocknumber not in self._block_timestamps:
# Try the local Eth server
logger.debug('Retrieving block #{}...'.format(blocknumber))
block = self.server.eth.getBlock(blocknumber,
full_transactions=False)
if block:
self._block_timestamps[blocknumber] = block.timestamp
else:
# Fallback to Etherchain.org
block = self.ec.api.get_block(3865982)
ts = self._parse_utc_time(block['time'])
self._block_timestamps[blocknumber] = ts
return self._block_timestamps[blocknumber]
@classmethod
def _parse_utc_time(cls, timestr):
dt = datetime.datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S.%fZ")
return calendar.timegm(dt.timetuple())
def perform_transaction(self, net_amount, miner_fee, target_address):
"""
Send Ether to target_address (total cost: net_amount + miner_fee)
"""
assert isinstance(net_amount, Decimal)
assert isinstance(miner_fee, Decimal)
net_amount_wei = self.server.toWei(net_amount, 'ether')
if net_amount_wei != net_amount * Decimal('1e18'):
raise ValueError('net_amount is not an integer multiple of wei.')
gas_price_wei = self._get_gas_price(miner_fee)
txid = self.server.personal.sendTransaction(
{
'to': target_address,
'gas': self.TXN_GAS_LIMIT,
'gasPrice': gas_price_wei,
'value': net_amount_wei,
}, self.wallet_password)
return txid
def _get_gas_price(self, eth_to_spend):
assert isinstance(eth_to_spend, Decimal)
wei_to_spend = self.server.toWei(eth_to_spend, 'ether')
gas_price_wei = wei_to_spend / self.TXN_GAS_LIMIT
if gas_price_wei != int(gas_price_wei):
raise ValueError(
'Transaction fee is not an integer multiple of the gas price.')
return int(gas_price_wei)