def request_get( url: str, timeout: int = ALL_REMOTES_TIMEOUT, handle_429: bool = False, backoff_in_seconds: Union[int, float] = 0, ) -> Union[Dict, List]: """ May raise: - UnableToDecryptRemoteData from request_get - Remote error if the get request fails """ # TODO make this a bit more smart. Perhaps conditional on the type of request. # Not all requests would need repeated attempts response = retry_calls( times=QUERY_RETRY_TIMES, location='', handle_429=handle_429, backoff_in_seconds=backoff_in_seconds, method_name=url, function=requests.get, # function's arguments url=url, timeout=timeout, ) try: result = rlk_jsonloads(response.text) except json.decoder.JSONDecodeError: raise UnableToDecryptRemoteData(f'{url} returned malformed json') return result
def request_get( url: str, timeout: int = ALL_REMOTES_TIMEOUT, handle_429: bool = False, backoff_in_seconds: Union[int, float] = 0, ) -> Union[Dict, List]: # TODO make this a bit more smart. Perhaps conditional on the type of request. # Not all requests would need repeated attempts response = retry_calls( times=QUERY_RETRY_TIMES, location='', handle_429=handle_429, backoff_in_seconds=backoff_in_seconds, method_name=url, function=requests.get, # function's arguments url=url, timeout=timeout, ) if response.status_code != 200: if 'https://blockchain.info/q/addressbalance' in url and response.status_code == 500: # For some weird reason blockchain.info returns # 500 server error when giving invalid account raise InvalidBTCAddress('Invalid BTC address given to blockchain.info') try: result = rlk_jsonloads(response.text) except json.decoder.JSONDecodeError: raise UnableToDecryptRemoteData(f'{url} returned malformed json') return result
def get_airdrop_data(name: str, data_dir: Path) -> Tuple[Iterator, TextIO]: airdrops_dir = data_dir / 'airdrops' airdrops_dir.mkdir(parents=True, exist_ok=True) filename = airdrops_dir / f'{name}.csv' if not filename.is_file(): # if not cached, get it from the gist try: response = requests.get(url=AIRDROPS[name][0], timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'Airdrops Gist request failed due to {str(e)}') from e try: content = response.text if (not csv.Sniffer().has_header(content) or len(response.content) < SMALLEST_AIRDROP_SIZE): raise csv.Error with open(filename, 'w') as f: f.write(content) except csv.Error as e: log.debug( f'airdrop file {filename} contains invalid data {content}') raise UnableToDecryptRemoteData( f'File {filename} contains invalid data. Check logs.', ) from e # Verify the CSV file csvfile = open(filename, 'r') iterator = csv.reader(csvfile) next(iterator) # skip header return iterator, csvfile
def check_airdrops( addresses: List[ChecksumEthAddress], data_dir: Path, ) -> Dict[ChecksumEthAddress, Dict]: """Checks airdrop data for the given list of ethereum addresses May raise: - RemoteError if the remote request fails """ found_data: Dict[ChecksumEthAddress, Dict] = defaultdict(lambda: defaultdict(dict)) for protocol_name, airdrop_data in AIRDROPS.items(): data, csvfile = get_airdrop_data(protocol_name, data_dir) for row in data: if len(row) < 2: raise UnableToDecryptRemoteData( f'Airdrop CSV for {protocol_name} contains an invalid row: {row}', ) addr, amount, *_ = row # not doing to_checksum_address() here since the file addresses are checksummed # and doing to_checksum_address() so many times hits performance if protocol_name in ('cornichon', 'tornado', 'grain', 'lido', 'sdl'): amount = token_normalized_value_decimals(int(amount), 18) if addr in addresses: found_data[addr][protocol_name] = { 'amount': str(amount), 'asset': airdrop_data[1], 'link': airdrop_data[2], } csvfile.close() # TODO: fix next line annotation for protocol_name, airdrop_data in POAP_AIRDROPS.items(): # type: ignore data_dict = get_poap_airdrop_data(protocol_name, data_dir) for addr, assets in data_dict.items(): # not doing to_checksum_address() here since the file addresses are checksummed # and doing to_checksum_address() so many times hits performance if addr in addresses: if 'poap' not in found_data[addr]: found_data[addr]['poap'] = [] found_data[addr]['poap'].append({ 'event': protocol_name, 'assets': assets, 'link': airdrop_data[1], 'name': airdrop_data[2], }) return dict(found_data)
def decrypt(key: bytes, given_source: str) -> bytes: """ Decrypts the given source data we with the given key. Returns the decrypted data. If data can't be decrypted then raises UnableToDecryptRemoteData """ assert isinstance(key, bytes), 'key should be given in bytes' assert isinstance(given_source, str), 'source should be given in string' source = base64.b64decode(given_source.encode("latin-1")) key = SHA256.new(key).digest() # use SHA-256 over our key to get a proper-sized AES key iv = source[:AES.block_size] # extract the iv from the beginning decryptor = AES.new(key, AES.MODE_CBC, iv) data = decryptor.decrypt(source[AES.block_size:]) # decrypt padding = data[-1] # pick the padding value from the end; Python 2.x: ord(data[-1]) if data[-padding:] != bytes([padding]) * padding: # Python 2.x: chr(padding) * padding raise UnableToDecryptRemoteData( 'Invalid padding when decrypting the DB data we received from the server. ' 'Are you using a new user and if yes have you used the same password as before? ' 'If you have then please open a bug report.', ) return data[:-padding] # remove the padding