def resolve_tracker_addresses(trackers):
"""
Given a list of Ethereum tracker URLs, returns a set of lowercased,
0x-prefixed Ethereum hex addresses.
"""
addresses = set()
for tracker in trackers:
path = urlparse(tracker).path
presumed_address = path.split('/')[-1]
if is_hex_address(presumed_address):
addresses.add(presumed_address.lower())
elif "etherscan.io" in tracker:
logging.debug(
"Trying to resolve '%s' as a custom etherscan.io address",
tracker)
redirect_url = get_etherescan_redirect_url(presumed_address)
resolved_address = redirect_url.strip('/').split('/')[-1]
if is_hex_address(resolved_address):
logging.debug("Resolved '%s' as %s", tracker, resolved_address)
addresses.add(resolved_address.lower())
else:
logging.error("Could not resolve etherscan.io: '%s'", tracker)
else:
logging.warn("Unknown tracker URL format: %s", tracker)
return set([to_checksum_address(address) for address in addresses])
def validate_validator_definition(validator_definition):
if not isinstance(validator_definition, Mapping):
raise ValueError("Validator definition must be a mapping")
if list(validator_definition.keys()) != ["multi"]:
raise ValueError("Validator definition must be multi list")
multi_list = validator_definition["multi"]
if not isinstance(multi_list, Mapping):
raise ValueError("Multi list must be a mapping")
if "0" not in multi_list:
raise ValueError(
"Multi list must contain validators for block number 0")
for multi_list_key, multi_list_entry in multi_list.items():
if not multi_list_key.isdigit():
raise ValueError("Multi list keys must be stringified ints")
if not isinstance(multi_list_entry, Mapping):
raise ValueError("Multi list entries must be a mapping")
if not len(multi_list_entry.keys()) == 1:
raise ValueError("Multi list entries must have exactly one key")
for multi_list_entry_type, multi_list_entry_data in multi_list_entry.items(
):
if multi_list_entry_type not in [
"list", "safeContract", "contract"
]:
raise ValueError(
"Multi list entries must be one of list, safeContract or contract"
)
if multi_list_entry_type == "list":
if not isinstance(multi_list_entry_data, list):
raise ValueError(
"Static validator list definition must be a list")
if len(multi_list_entry_data) < 1:
raise ValueError("Static validator list must not be empty")
if any(not is_hex_address(address)
for address in multi_list_entry_data):
raise ValueError(
"Static validator list must only contain hex addresses"
)
elif multi_list_entry_type in ["safeContract", "contract"]:
if not is_hex_address(multi_list_entry_data):
raise ValueError(
"Validator contract address must be a single hex address"
)
else:
assert (
False
), "Unreachable. Multi list entry type has already been validated."
def iou_side_effect(*_, **kwargs):
assert "params" in kwargs
body = kwargs["params"]
assert is_hex_address(body["sender"])
assert is_hex_address(body["receiver"])
assert "timestamp" in body
assert is_hex(body["signature"])
assert len(body["signature"]) == 65 * 2 + 2 # 65 hex encoded bytes with 0x prefix
return mocked_json_response(response_data=iou_json_data)
def iou_side_effect(*_, **kwargs):
assert "params" in kwargs
body = kwargs["params"]
assert is_hex_address(body["sender"])
assert is_hex_address(body["receiver"])
assert "timestamp" in body
assert is_hex(body["signature"])
assert len(body["signature"]
) == 65 * 2 + 2 # 65 hex encoded bytes with 0x prefix
return Mock(json=Mock(return_value=iou_json_data or {}),
status_code=200)
def iou_side_effect(*_, **kwargs):
assert 'data' in kwargs
body = kwargs['data']
assert is_hex_address(body['sender'])
assert is_hex_address(body['receiver'])
assert 'timestamp' in body
assert is_hex(body['signature'])
assert len(body['signature']) == 66
return Mock(
json=Mock(return_value=iou_json_data or {}),
status_code=200,
)
def validate_address(value):
"""
Helper function for validating an address
"""
if is_bytes(value):
if not is_binary_address(value):
raise InvalidAddress("Address must be 20 bytes when input type is bytes", value)
return
if not isinstance(value, str):
raise TypeError('Address {} must be provided as a string'.format(value))
if not is_hex_address(value):
raise InvalidAddress("Address must be 20 bytes, as a hex string with a 0x prefix", value)
if not is_checksum_address(value):
if value == value.lower():
raise InvalidAddress(
"Web3.py only accepts checksum addresses. "
"The software that gave you this non-checksum address should be considered unsafe, "
"please file it as a bug on their platform. "
"Try using an ENS name instead. Or, if you must accept lower safety, "
"use Web3.toChecksumAddress(lower_case_address).",
value,
)
else:
raise InvalidAddress(
"Address has an invalid EIP-55 checksum. "
"After looking up the address from the original source, try again.",
value,
)
def validate_account(value):
if not is_text(value):
raise ValidationError("Address must be 20 bytes encoded as hexidecimal")
elif not is_hex_address(value):
raise ValidationError("Address must be 20 bytes encoded as hexidecimal")
elif is_checksum_formatted_address(value) and not is_checksum_address(value):
raise ValidationError("Address does not validate EIP55 checksum")
def validate_filter_params(from_block, to_block, address, topics):
# blocks
if from_block is not None:
validate_block_number(from_block)
if to_block is not None:
validate_block_number(to_block)
# address
if address is None:
pass
elif is_list_like(address):
if not address:
raise ValidationError(
"Address must be either a single hexidecimal encoded address or "
"a non-empty list of hexidecimal encoded addresses")
for sub_address in address:
validate_account(sub_address)
elif not is_hex_address(address):
validate_account(address)
invalid_topics_message = (
"Topics must be one of `None`, an array of 32 byte hexidecimal encoded "
"strings, or an array of arrays of 32 byte hexidecimal strings")
# topics
if topics is None:
pass
elif not is_list_like(topics):
raise ValidationError(invalid_topics_message)
elif is_valid_topic_array(topics):
return True
else:
raise ValidationError(invalid_topics_message)
def validate_address(value):
"""
Helper function for validating an address
"""
if is_bytes(value):
if not is_binary_address(value):
raise InvalidAddress(
"Address must be 20 bytes when input type is bytes", value)
return
if not isinstance(value, str):
raise TypeError(
'Address {} must be provided as a string'.format(value))
if not is_hex_address(value):
raise InvalidAddress(
"Address must be 20 bytes, as a hex string with a 0x prefix",
value)
if not is_checksum_address(value):
if value == value.lower():
raise InvalidAddress(
"Web3.py only accepts checksum addresses. "
"The software that gave you this non-checksum address should be considered unsafe, "
"please file it as a bug on their platform. "
"Try using an ENS name instead. Or, if you must accept lower safety, "
"use Web3.toChecksumAddress(lower_case_address).",
value,
)
else:
raise InvalidAddress(
"Address has an invalid EIP-55 checksum. "
"After looking up the address from the original source, try again.",
value,
)
def clean_contract_address(self):
contract_address = self.cleaned_data['contract_address']
if not is_hex_address(contract_address):
raise forms.ValidationError(
_('Contract address %(address)s is not a valid hex address') %
{'address': contract_address})
return Web3.toChecksumAddress(contract_address)
def is_ens_name(value):
if not isinstance(value, str):
return False
elif is_hex_address(value):
return False
elif is_0x_prefixed(value) and is_hex(value):
return False
else:
return ENS.is_valid_name(value)
def is_ens_name(value):
if not isinstance(value, str):
return False
elif is_hex_address(value):
return False
elif is_0x_prefixed(value) and is_hex(value):
return False
else:
return ENS.is_valid_name(value)
def validate_eth1_withdrawal_address(cts: click.Context, param: Any,
address: str) -> HexAddress:
if address is None:
return None
if not is_hex_address(address):
raise ValueError(load_text(['err_invalid_ECDSA_hex_addr']))
normalized_address = to_normalized_address(address)
click.echo('\n%s\n' % load_text(['msg_ECDSA_addr_withdrawal']))
return normalized_address
def validate_address(value):
"""
Helper function for validating an address
"""
if not isinstance(value, str):
raise TypeError('Address {} must be provided as a string'.format(value))
if not is_hex_address(value):
raise InvalidAddress("Address must be 20 bytes, as a hex string with a 0x prefix", value)
if not is_checksum_address(value):
raise InvalidAddress("Address has an invalid EIP checksum", value)
def get_etherscan_contract_address(addr, html_doc=None):
if not html_doc:
html_doc = get_etherscan_token_page(addr)
soup = BeautifulSoup(html_doc, 'html.parser')
selector = "#ContentPlaceHolder1_trContract a"
try:
eth_address = soup.select(selector)[0].text.strip()
assert (is_hex_address(eth_address))
return eth_address
except (IndexError, AssertionError):
return None
def generate_keys(args):
"""Generate validator keys.
Keyword arguments:
args -- contains the CLI arguments pass to the application, it should have those properties:
- wordlist: path to the word lists directory
- mnemonic: mnemonic to be used as the seed for generating the keys
- index: index of the first validator's keys you wish to generate
- count: number of signing keys you want to generate
- folder: folder path for the resulting keystore(s) and deposit(s) files
- network: network setting for the signing domain, possible values are 'mainnet',
'prater', 'kintsugi' or 'kiln'
- password: password that will protect the resulting keystore(s)
- eth1_withdrawal_address: (Optional) eth1 address that will be used to create the
withdrawal credentials
"""
eth1_withdrawal_address = None
if args.eth1_withdrawal_address:
eth1_withdrawal_address = args.eth1_withdrawal_address
if not is_hex_address(eth1_withdrawal_address):
raise ValueError(
"The given Eth1 address is not in hexadecimal encoded form.")
eth1_withdrawal_address = to_normalized_address(
eth1_withdrawal_address)
mnemonic = validate_mnemonic(args.mnemonic, args.wordlist)
mnemonic_password = ''
amounts = [MAX_DEPOSIT_AMOUNT] * args.count
folder = args.folder
chain_setting = get_chain_setting(args.network)
if not os.path.exists(folder):
os.mkdir(folder)
credentials = CredentialList.from_mnemonic(
mnemonic=mnemonic,
mnemonic_password=mnemonic_password,
num_keys=args.count,
amounts=amounts,
chain_setting=chain_setting,
start_index=args.index,
hex_eth1_withdrawal_address=eth1_withdrawal_address,
)
keystore_filefolders = credentials.export_keystores(password=args.password,
folder=folder)
deposits_file = credentials.export_deposit_data_json(folder=folder)
if not credentials.verify_keystores(
keystore_filefolders=keystore_filefolders, password=args.password):
raise ValidationError("Failed to verify the keystores.")
if not verify_deposit_data_json(deposits_file, credentials.credentials):
raise ValidationError("Failed to verify the deposit data JSON files.")
def main(listings):
for listing in listings:
print("Fetching", listing["id"])
if not has_cached_coin_details(listing["id"]):
sleep(0.5)
coin_details = fetch_coin_details(listing["id"])
if is_hex_address(coin_details.get("contract_address")):
# We've got a live one!
token_entry = make_token_entry(coin_details)
write_token_entry(token_entry["address"], token_entry)
def encode_topic(topic):
try:
if isinstance(topic, bytes) or is_hex_address(topic):
return to_checksum_address(topic)
if isinstance(topic, str) or is_checksum_address(topic):
return topic
raise TypeError
except:
raise TypeError(
"encode_topic requires address representation either in 0x prefixed String or valid bytes format \
as argument. Got: {0}".format(repr(topic)))
def iou_side_effect(*args, **kwargs):
if args[0].endswith("/info"):
return mocked_json_response({
"price_info":
5,
"network_info": {
"chain_id":
42,
"token_network_registry_address":
to_checksum_address(
factories.make_token_network_registry_address()),
"user_deposit_address":
to_checksum_address(factories.make_address()),
"confirmed_block": {
"number": 11
},
},
"version":
"0.0.3",
"operator":
"John Doe",
"message":
"This is your favorite pathfinding service",
"payment_address":
to_checksum_address(factories.make_address()),
"matrix_server":
"http://matrix.example.com",
})
else:
assert "params" in kwargs
body = kwargs["params"]
assert is_hex_address(body["sender"])
assert is_hex_address(body["receiver"])
assert "timestamp" in body
assert is_hex(body["signature"])
assert len(body["signature"]
) == 65 * 2 + 2 # 65 hex encoded bytes with 0x prefix
return mocked_json_response(response_data=iou_json_data)
def validate_address(value):
"""
Helper function for validating an address
"""
if not isinstance(value, str):
raise TypeError('Address {} must be provided as a string'.format(value))
if not is_hex_address(value):
raise InvalidAddress("Address must be 20 bytes, as a hex string with a 0x prefix", value)
if not is_checksum_address(value):
# Shamelessly ignore EIP checksum validation
# https://github.com/ethereum/web3.py/issues/674
# raise InvalidAddress("Address has an invalid EIP checksum", value)
pass
def validate_address(value):
"""
Helper function for validating an address
"""
if not isinstance(value, str):
raise TypeError(
'Address {} must be provided as a string'.format(value))
if not is_hex_address(value):
raise InvalidAddress(
"Address must be 20 bytes, as a hex string with a 0x prefix",
value)
if not is_checksum_address(value):
pass
def validate_eth1_withdrawal_address(cts: click.Context, param: Any,
address: str) -> HexAddress:
if address is None:
return None
if not is_hex_address(address):
raise ValueError(
"The given Eth1 address is not in hexadecimal encoded form.")
normalized_address = to_normalized_address(address)
click.echo(
f'\n**[Warning] you are setting Eth1 address {normalized_address} as your withdrawal address. '
'Please ensure that you have control over this address.**\n')
return normalized_address
def main(listings):
for listing in listings:
print("Fetching", listing["id"])
if not has_cached_coin_details(listing["id"]):
sleep(0.5)
coin_details = fetch_coin_details(listing["id"])
token_platform = coin_details.get("asset_platform_id")
token_address = coin_details.get("contract_address")
if token_platform == "ethereum" and is_hex_address(token_address):
# We've got a live one!
token_entry = make_token_entry(coin_details)
write_token_entry(token_entry["address"], token_entry)
def validate_ethereum_address(address: str):
"""Clever Ethereum address validator."""
if len(address) < 42:
raise ValueError("Not an Ethereum address: {}".format(address))
try:
if not is_hex_address(address):
raise ValueError("Not an Ethereum address: {}".format(address))
except UnicodeEncodeError:
raise ValueError("Could not decode: {}".format(address))
# Check if checksummed address if any of the letters is upper case
if any([c.isupper() for c in address]):
if not is_checksum_address(address):
raise ValueError(
"Not a checksummed Ethereum address: {}".format(address))
def validate(self):
for field in fields(self):
type_ = dict if field.name in ('bridge_addresses',
'reward_thresholds',
'ui') else field.type
value = getattr(self, field.name, None)
if value is None:
raise ValueError(f'missing value for {field.name}')
if not isinstance(value, type_):
raise ValueError(
f'expected {field.name} to be of type {type_}, was {type(value)}'
)
for bridge_key, bridge_address in self.bridge_addresses.items():
if not is_hex_address(bridge_address):
raise ValueError(
f'address {bridge_address!r} for bridge {bridge_key!r} is not a valid hex address'
)
if self.reward_rbtc > Decimal('0.1'):
raise ValueError(
f'RBTC reward amount {str(self.reward_rbtc)} is dangerously high. '
'Did you accidentally pass in the amount as WEI instead of decimal?'
)
if self.deposit_fee_percentage > Decimal('0.1'):
raise ValueError(
f'Invalid deposit fee percentage {str(self.deposit_fee_percentage)} '
'Cannot be over 10% (0.1).')
if not self.reward_thresholds:
raise ValueError(
'Empty reward_thresholds -- no rewards would be given')
for key, value in self.reward_thresholds.items():
if not isinstance(key, str):
raise ValueError(
'expected reward_threshold keys to be strings (token symbols)'
)
if not isinstance(value, Decimal):
raise ValueError(
'expected reward_threshold values to be Decimals (amounts)'
)
def _validate_inbound_access_list(access_list):
"""
Validates the structure of an inbound access list. This is similar to the JSON-RPC structure
for an access list only with `under_score` keys rather than `camelCase`.
>>> _access_list = (
... {
... 'address': '0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae',
... 'storage_keys': (
... '0x0000000000000000000000000000000000000000000000000000000000000003',
... '0x0000000000000000000000000000000000000000000000000000000000000007',
... )
... },
... {
... 'address': '0xbb9bc244d798123fde783fcc1c72d3bb8c189413',
... 'storage_keys': ()
... },
... )
"""
if not is_list_like(access_list):
raise ValidationError('access_list is not list-like')
for entry in access_list:
if not is_dict(entry) and len(entry) != 2:
raise ValidationError(
f'access_list entry not properly formatted: {entry}')
address = entry.get('address')
storage_keys = entry.get('storage_keys')
if not is_hex_address(address):
raise ValidationError(
f'access_list address must be a hexadecimal address: {address}'
)
if not is_list_like(storage_keys):
raise ValidationError(
f'access_list storage keys are not list-like: {storage_keys}')
if len(storage_keys) > 0 and not all(
is_32byte_hex_string(k) for k in storage_keys):
raise ValidationError(
f'one or more access list storage keys not formatted properly: {storage_keys}'
)
def get_ethereum_addresses(next_data):
"""
Returns a set of Ethereum addresses for a list of explorer links. The
set may be empty if the CoinMarketCap listing does not link to any supported
Ethereum chain browsers. The set may contain more than one element if the
CoinMarketCap listing contains tracker URLs that resolve to different
Ethereum addresses.
Ethereum addresses are returned in checksummed 0x-prefixed hex format.
"""
tracker_url_predicate = lambda url: any(True for s in KNOWN_TRACKER_SUFFIXES if s in url)
base_data = get_asset_data(next_data, "info")
explorer_links = base_data["urls"]["explorer"]
ethereum_addresses = resolve_tracker_addresses(
filter(tracker_url_predicate, explorer_links))
if base_data["platform"] and base_data["platform"]["name"] == "Ethereum":
token_address = base_data["platform"]["token_address"]
if is_hex_address(token_address):
ethereum_addresses.add(to_checksum_address(token_address))
return ethereum_addresses
def recover_to_addr(token, signature):
msghash = defunct_hash_message(text=token)
address = w3.eth.account.recoverHash(msghash, signature=signature)
res = is_hex_address(address)
return address
def is_convertible(self, value: str) -> bool:
return isinstance(value, str) and is_hex_address(value) and not is_checksum_address(value)
def test_addr_ethereum_address(content):
"addr must be a valid Ethereum address starting with '0x'"
assert is_hex_address(content["addr"]), \
"`addr` is not a valid Ethereum address"
assert content["addr"].startswith("0x"), \
"expected `addr` to start with '0x', but got '{}'".format(content["addr"][:2])
def validate_eth_address(value):
if not is_hex_address(value):
raise forms.ValidationError(
_('%(value)s is not a valid Ethereum address'),
params={'value': value},
)
def validate_0x_prefixed_hex_address(field, value, error):
if not is_hex_address(value) or not is_0x_prefixed(value):
error(field, 'must be a 0x-prefixed hex Ethereum address')
def read_file(combined: dict, all_errors: List[tuple], book_keeping: collections.Counter, csv_file: str, decimals: int, address_column: str, amount_column: str):
"""Process a single CSV file."""
rounder = Decimal(10) ** (-1 * decimals)
sum = rounded_sum = Decimal(0)
errors = []
print("Reading file:", csv_file, "with rounder", rounder)
with open(csv_file, "rt", encoding="utf-8", errors='ignore') as inp:
reader = csv.DictReader(inp)
rows = [row for row in reader]
# First check all rows
good_rows = [] # type: List[dict]
for line, row in enumerate(rows, start=1):
address = row[address_column]
amount = row[amount_column]
address = address.strip()
amount = amount.strip()
# Check for any Ethereum address
try:
if not is_hex_address(address):
errors.append((csv_file, line, "Not an Ethereum address: {}".format(address)))
continue
except UnicodeEncodeError:
errors.append([csv_file, line, "Could not decode: {}".format(address)])
continue
# Check if checksummed address if any of the letters is upper case
if any([c.isupper() for c in address]):
if not is_checksum_address(address):
errors.append((csv_file, line, "Not a checksummed Ethereum address: {}".format(address)))
continue
try:
amount = Decimal(amount)
except ValueError:
errors.append((csv_file, line, "Bad decimal amount: {}".format(amount)))
continue
good_rows.append(row)
# Then do a full pass on rows where users did not crap their data
for row in good_rows:
address = row[address_column].strip()
amount = row[amount_column].strip()
amount = Decimal(amount)
rounded_amount = amount.quantize(rounder, rounding=ROUND_HALF_DOWN) # Use explicit rounding
sum += amount
rounded_sum += rounded_amount
# Make sure we use the same format for the addresses everywehre
address = address.lower()
entry = combined.get(address)
if not entry:
entry = AddressEntry(amount=Decimal(0), rounded_amount=Decimal(0), sources=set(), address_forms=set(), count=0)
combined[address] = entry
book_keeping["uniq_entries"] += 1
entry.sources.add(csv_file)
entry.address_forms.add(row[address_column]) # Record original spelling of the address
entry.amount += amount
entry.rounded_amount += rounded_amount
book_keeping["token_total"] += rounded_amount
entry.count += 1
book_keeping["total_entries"] += 1
all_errors += errors
print("File:", csv_file, "total sum", sum)
print("File:", csv_file, "rounded sum", rounded_sum)
print("File:", csv_file, "errors", len(errors))
def test_addr_ethereum_address(content):
"addr must be a valid Ethereum address starting with '0x'"
assert is_hex_address(content["addr"]), \
"`addr` is not a valid Ethereum address"
assert content["addr"].startswith("0x"), \
"expected `addr` to start with '0x', but got '{}'".format(content["addr"][:2])
def read_file(combined: dict, all_errors: List[tuple],
book_keeping: collections.Counter, csv_file: str, decimals: int,
address_column: str, amount_column: str):
"""Process a single CSV file."""
rounder = Decimal(10)**(-1 * decimals)
sum = rounded_sum = Decimal(0)
errors = []
print("Reading file:", csv_file, "with rounder", rounder)
with open(csv_file, "rt", encoding="utf-8", errors='ignore') as inp:
reader = csv.DictReader(inp)
rows = [row for row in reader]
# First check all rows
good_rows = [] # type: List[dict]
for line, row in enumerate(rows, start=1):
address = row[address_column]
amount = row[amount_column]
address = address.strip()
amount = amount.strip()
# Check for any Ethereum address
if len(address) < 42:
errors.append((csv_file, line,
"Not an Ethereum address: {}".format(address)))
continue
try:
if not is_hex_address(address):
errors.append(
(csv_file, line,
"Not an Ethereum address: {}".format(address)))
continue
except UnicodeEncodeError:
errors.append(
[csv_file, line, "Could not decode: {}".format(address)])
continue
# Check if checksummed address if any of the letters is upper case
if any([c.isupper() for c in address]):
if not is_checksum_address(address):
errors.append(
(csv_file, line,
"Not a checksummed Ethereum address: {}".format(
address)))
continue
try:
amount = Decimal(amount)
except ValueError:
errors.append(
(csv_file, line, "Bad decimal amount: {}".format(amount)))
continue
good_rows.append(row)
# Then do a full pass on rows where users did not crap their data
for row in good_rows:
address = row[address_column].strip()
amount = row[amount_column].strip()
amount = Decimal(amount)
rounded_amount = amount.quantize(
rounder, rounding=ROUND_HALF_DOWN) # Use explicit rounding
sum += amount
rounded_sum += rounded_amount
# Make sure we use the same format for the addresses everywehre
address = address.lower()
entry = combined.get(address)
if not entry:
entry = AddressEntry(amount=Decimal(0),
rounded_amount=Decimal(0),
sources=set(),
address_forms=set(),
count=0)
combined[address] = entry
book_keeping["uniq_entries"] += 1
entry.sources.add(csv_file)
entry.address_forms.add(
row[address_column]) # Record original spelling of the address
entry.amount += amount
entry.rounded_amount += rounded_amount
book_keeping["token_total"] += rounded_amount
entry.count += 1
book_keeping["total_entries"] += 1
all_errors += errors
print("File:", csv_file, "total sum", sum)
print("File:", csv_file, "rounded sum", rounded_sum)
print("File:", csv_file, "errors", len(errors))
