def request_get(
url: str,
timeout: int = ALL_REMOTES_TIMEOUT,
handle_429: bool = False,
backoff_in_seconds: Union[int, float] = 0,
) -> Union[Dict, List]:
"""
May raise:
- UnableToDecryptRemoteData from request_get
- Remote error if the get request fails
"""
# TODO make this a bit more smart. Perhaps conditional on the type of request.
# Not all requests would need repeated attempts
response = retry_calls(
times=QUERY_RETRY_TIMES,
location='',
handle_429=handle_429,
backoff_in_seconds=backoff_in_seconds,
method_name=url,
function=requests.get,
# function's arguments
url=url,
timeout=timeout,
)
try:
result = rlk_jsonloads(response.text)
except json.decoder.JSONDecodeError:
raise UnableToDecryptRemoteData(f'{url} returned malformed json')
return result
def request_get(
url: str,
timeout: int = ALL_REMOTES_TIMEOUT,
handle_429: bool = False,
backoff_in_seconds: Union[int, float] = 0,
) -> Union[Dict, List]:
# TODO make this a bit more smart. Perhaps conditional on the type of request.
# Not all requests would need repeated attempts
response = retry_calls(
times=QUERY_RETRY_TIMES,
location='',
handle_429=handle_429,
backoff_in_seconds=backoff_in_seconds,
method_name=url,
function=requests.get,
# function's arguments
url=url,
timeout=timeout,
)
if response.status_code != 200:
if 'https://blockchain.info/q/addressbalance' in url and response.status_code == 500:
# For some weird reason blockchain.info returns
# 500 server error when giving invalid account
raise InvalidBTCAddress('Invalid BTC address given to blockchain.info')
try:
result = rlk_jsonloads(response.text)
except json.decoder.JSONDecodeError:
raise UnableToDecryptRemoteData(f'{url} returned malformed json')
return result
def get_airdrop_data(name: str, data_dir: Path) -> Tuple[Iterator, TextIO]:
airdrops_dir = data_dir / 'airdrops'
airdrops_dir.mkdir(parents=True, exist_ok=True)
filename = airdrops_dir / f'{name}.csv'
if not filename.is_file():
# if not cached, get it from the gist
try:
response = requests.get(url=AIRDROPS[name][0],
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Airdrops Gist request failed due to {str(e)}') from e
try:
content = response.text
if (not csv.Sniffer().has_header(content)
or len(response.content) < SMALLEST_AIRDROP_SIZE):
raise csv.Error
with open(filename, 'w') as f:
f.write(content)
except csv.Error as e:
log.debug(
f'airdrop file {filename} contains invalid data {content}')
raise UnableToDecryptRemoteData(
f'File {filename} contains invalid data. Check logs.', ) from e
# Verify the CSV file
csvfile = open(filename, 'r')
iterator = csv.reader(csvfile)
next(iterator) # skip header
return iterator, csvfile
def check_airdrops(
addresses: List[ChecksumEthAddress],
data_dir: Path,
) -> Dict[ChecksumEthAddress, Dict]:
"""Checks airdrop data for the given list of ethereum addresses
May raise:
- RemoteError if the remote request fails
"""
found_data: Dict[ChecksumEthAddress,
Dict] = defaultdict(lambda: defaultdict(dict))
for protocol_name, airdrop_data in AIRDROPS.items():
data, csvfile = get_airdrop_data(protocol_name, data_dir)
for row in data:
if len(row) < 2:
raise UnableToDecryptRemoteData(
f'Airdrop CSV for {protocol_name} contains an invalid row: {row}',
)
addr, amount, *_ = row
# not doing to_checksum_address() here since the file addresses are checksummed
# and doing to_checksum_address() so many times hits performance
if protocol_name in ('cornichon', 'tornado', 'grain', 'lido',
'sdl'):
amount = token_normalized_value_decimals(int(amount), 18)
if addr in addresses:
found_data[addr][protocol_name] = {
'amount': str(amount),
'asset': airdrop_data[1],
'link': airdrop_data[2],
}
csvfile.close()
# TODO: fix next line annotation
for protocol_name, airdrop_data in POAP_AIRDROPS.items(): # type: ignore
data_dict = get_poap_airdrop_data(protocol_name, data_dir)
for addr, assets in data_dict.items():
# not doing to_checksum_address() here since the file addresses are checksummed
# and doing to_checksum_address() so many times hits performance
if addr in addresses:
if 'poap' not in found_data[addr]:
found_data[addr]['poap'] = []
found_data[addr]['poap'].append({
'event': protocol_name,
'assets': assets,
'link': airdrop_data[1],
'name': airdrop_data[2],
})
return dict(found_data)
def decrypt(key: bytes, given_source: str) -> bytes:
"""
Decrypts the given source data we with the given key.
Returns the decrypted data.
If data can't be decrypted then raises UnableToDecryptRemoteData
"""
assert isinstance(key, bytes), 'key should be given in bytes'
assert isinstance(given_source, str), 'source should be given in string'
source = base64.b64decode(given_source.encode("latin-1"))
key = SHA256.new(key).digest() # use SHA-256 over our key to get a proper-sized AES key
iv = source[:AES.block_size] # extract the iv from the beginning
decryptor = AES.new(key, AES.MODE_CBC, iv)
data = decryptor.decrypt(source[AES.block_size:]) # decrypt
padding = data[-1] # pick the padding value from the end; Python 2.x: ord(data[-1])
if data[-padding:] != bytes([padding]) * padding: # Python 2.x: chr(padding) * padding
raise UnableToDecryptRemoteData(
'Invalid padding when decrypting the DB data we received from the server. '
'Are you using a new user and if yes have you used the same password as before? '
'If you have then please open a bug report.',
)
return data[:-padding] # remove the padding