def get_block_by_hash(self, block_hash: str) -> Union[Dict[str, str], None]: """ Retrieves block information specified by its hash. Args: block_hash: Block hash. Returns: Dictionary representing the blockchain block. """ block_index = db_get_wrapper(self.db, b'hash-block-' + block_hash.encode()) if block_index is None: LOG.info('Block of specified hash not found.') return None raw_block = db_get_wrapper(self.db, b'block-' + block_index) block = coder.decode_block(raw_block) block['number'] = block_index.decode() transaction_hashes = block['transactions'].split('+') transactions = [] # type: List[Dict[str, Any]] if transaction_hashes == ['']: block['transactions'] = transactions return block for tx_hash in transaction_hashes: transaction = self.get_transaction_by_hash(tx_hash) transactions.append(transaction) # type: ignore block['transactions'] = transactions return block
def get_transactions_of_block_by_hash( self, block_hash: str) -> Union[List[Dict[str, Any]], None]: """ Gets list of transactions belonging to specified block. Args: block_hash: hash of the block. Returns: List of specified block transactions. """ block_index = db_get_wrapper(self.db, b'hash-block-' + block_hash.encode()) if block_index is None: LOG.info('Block of specified hash not found.') return None raw_block = db_get_wrapper(self.db, b'block-' + block_index) block = coder.decode_block(raw_block) transaction_hashes = block['transactions'].split('+') transactions = [] # type: List[Dict[str, Any]] if transaction_hashes == ['']: block['transactions'] = transactions return [] for tx_hash in transaction_hashes: transaction = self.get_transaction_by_hash(tx_hash) transactions.append(transaction) # type: ignore return transactions
def get_transaction_by_hash(self, tx_hash) -> Union[Dict[str, Any], None]: """ Retrieves transaction specified by its hash. Args: tx_hash: Hash of the transaction. Returns: The desired transaction. """ raw_tx = db_get_wrapper(self.db, b'transaction-' + tx_hash.encode()) if raw_tx is None: LOG.info('Transaction of given hash not found.') return None transaction = coder.decode_transaction(raw_tx) transaction['hash'] = tx_hash internal_tx_indexes = [] # type: List[Any] if transaction['internalTxIndex'] > 0: prefix = 'associated-data-' + tx_hash + '-tit-' print('OOO') internal_tx_indexes = db_iter_wrapper(self.db, prefix) print('OOOO') internal_transactions = [] for tx_index in internal_tx_indexes: tx_decoded = tx_index.decode() raw_tx = db_get_wrapper(self.db, b'internal-tx-' + tx_decoded.encode()) int_tx = coder.decode_internal_tx(raw_tx) internal_transactions.append(int_tx) transaction.pop('internalTxIndex', None) transaction['internalTransactions'] = internal_transactions return transaction
def expand_tokens(self, tokens: Dict, token_txs: List) -> Tuple[Dict, Dict]: """ Find all relevant tokens from DB and reject not-found transactions. Args: tokens: Tokens gathered so far (in this batch). token_txs: Token transactions to get other token info. Returns: Updated token list. """ full_tokens = {} filtered_txs = {} for token_tx in token_txs: data = db_get_wrapper( self.db, b'token-' + token_tx['tokenAddress'].encode()) if data is not None: db_token = coder.decode_token(data) db_token['transactions'] = [] full_tokens[token_tx['tokenAddress']] = db_token self._highest_token_tx += 1 filtered_txs[self._highest_token_tx] = token_tx elif token_tx['tokenAddress'] in tokens: full_tokens[token_tx['tokenAddress']] = tokens[ token_tx['tokenAddress']] self._highest_token_tx += 1 filtered_txs[self._highest_token_tx] = token_tx for token in tokens: if token not in full_tokens: full_tokens[token] = tokens[token] return (full_tokens, filtered_txs)
def get_transactions_of_block_by_index( self, block_index: str) -> Union[List[Dict[str, Any]], None]: """ Gets list of transactions belonging to specified block. Args: block_index: index of the block. Returns: List of specified block transactions. """ raw_block = db_get_wrapper(self.db, b'block-' + block_index.encode()) block = coder.decode_block(raw_block) transaction_hashes = block['transactions'].split('+') transactions = [] # type: List[Dict[str, Any]] if transaction_hashes == ['']: block['transactions'] = transactions return [] for tx_hash in transaction_hashes: transaction = self.get_transaction_by_hash(tx_hash) transactions.append(transaction) # type: ignore return transactions
def get_token(self, addr: str, time_from: int, time_to: int, no_tx_list: int) -> Union[Dict[str, Any], None]: """ Get informtion about a token based on its contract address. Args: addr: Token address. time_from: Beginning datetime to take transactions from. time_to: Ending datetime to take transactions from. no_tx_list: Maximum transactions to return. Returns: Information about a token. """ raw_token = db_get_wrapper(self.db, b'token-' + addr.encode()) if raw_token is None: return None token = coder.decode_token(raw_token) token['address'] = addr token_tx_indexes = [] # type: List[bytes] if token['txIndex'] > 0: prefix = 'associated-data-' + addr + '-tt-' token_tx_indexes = db_iter_wrapper(self.db, prefix) found_txs = 0 token_txs = [] for token_tx_index in token_tx_indexes: if found_txs >= no_tx_list: break tx_decoded = token_tx_index.decode() tx_index, timestamp = tx_decoded.split('-') if (time_from <= int(timestamp) and time_to >= int(timestamp)): raw_tx = db_get_wrapper(self.db, b'token-tx-' + tx_index.encode()) token_tx = coder.decode_token_tx(raw_tx) token_txs.append(token_tx) found_txs += 1 token.pop('txIndex', None) token['tokenTransactions'] = token_txs return token
def get_block_hash_by_index(self, block_index: str) -> Union[str, None]: """ Retrieves block hash by its index. Args: block_index: Index of a block. Returns: Block hash. """ raw_block = db_get_wrapper(self.db, b'block-' + block_index.encode()) if raw_block is None: return None block = coder.decode_block(raw_block) return block['hash']
def get_balance(self, addr: str) -> Union[str, None]: """ Get balance of an address. Args: addr: Requested address. Returns: Current balance of an address. """ raw_address = db_get_wrapper(self.db, b'address-' + addr.encode()) if raw_address is None: return None address = coder.decode_address(raw_address) return address['balance']
def _update_db_balances(self, addr_balances: Dict) -> None: """ Updates balances of Ethereum addresses in the LevelDB database in batches. Args: addr_balances: Dictionary containing 'address: balance' entries. """ address_objects = {} for address in addr_balances: raw_addr = db_get_wrapper(self.db, b'address-' + str(address).encode()) if raw_addr is None: continue address_objects[address] = coder.decode_address(raw_addr) address_objects[address]['balance'] = addr_balances[address] self.db_lock.acquire() wb = rocksdb.WriteBatch() for address in address_objects: address_value = coder.encode_address(address_objects[address]) wb.put(b'address-' + str(address).encode(), address_value) self.db.write(wb) self.db_lock.release()
def get_blocks_by_indexes( self, index_start: str, index_end: str) -> Union[List[Dict[str, Any]], None]: """ Retrieves a list of blocks specified by an index range. Args: index_start: First index. index_end: Last index. Returns: A list of blocks. """ blocks = [] for block_index in range(int(index_start), int(index_end) + 1): raw_block = db_get_wrapper(self.db, b'block-' + str(block_index).encode()) block = coder.decode_block(raw_block) block['number'] = block_index transaction_hashes = block['transactions'].split('+') transactions = [] # type: List[Dict[str, Any]] if transaction_hashes == ['']: block['transactions'] = transactions blocks.append(block) continue for tx_hash in transaction_hashes: transaction = self.get_transaction_by_hash(tx_hash) transactions.append(transaction) # type: ignore block['transactions'] = transactions blocks.append(block) return blocks
def fill_addresses(self, addresses: Dict, transactions: Dict, tokens: Dict, token_txs: List) -> Tuple[Dict, Dict]: """ Fill addresses with transaction information. Args: addresses: Currently processed addresses. transactions: Currently processed transactions. tokens: Currently processed tokens. token_txs: Currently processed token transactions. Returns: Addresses with new information. """ LOG.info('Filling addresses.') updated_tokens, filtered_token_txs = self.expand_tokens( tokens, token_txs) addresses, updated_tokens = self.fill_addresses_tokens( addresses, updated_tokens, filtered_token_txs) addresses_encode = {} addresses_write_dict = {} for addr_hash, addr_dict in addresses.items(): existing_data = db_get_wrapper(self.db, b'address-' + addr_hash.encode()) # Address not yet in records if existing_data is not None: existing_address = coder.decode_address(existing_data) last_input_tx_index = int(existing_address['inputTxIndex']) last_output_tx_index = int(existing_address['outputTxIndex']) last_input_token_tx_index = int( existing_address['inputTokenTxIndex']) last_output_token_tx_index = int( existing_address['outputTokenTxIndex']) last_input_int_tx_index = int( existing_address['inputIntTxIndex']) last_output_int_tx_index = int( existing_address['outputIntTxIndex']) last_mined_block_index = int(existing_address['minedIndex']) else: last_input_tx_index = 0 last_output_tx_index = 0 last_input_token_tx_index = 0 last_output_token_tx_index = 0 last_input_int_tx_index = 0 last_output_int_tx_index = 0 last_mined_block_index = 0 address_encode = {} if existing_data is not None: address_encode['tokenContract'] = existing_address[ 'tokenContract'] if addr_hash in updated_tokens: updated_tokens[addr_hash]['type'] = existing_address[ 'tokenContract'] else: if 'tokenContract' in addr_dict: address_encode['tokenContract'] = addr_dict[ 'tokenContract'] if addr_hash in updated_tokens: updated_tokens[addr_hash]['type'] = addr_dict[ 'tokenContract'] else: address_encode['tokenContract'] = 'False' address_encode['balance'] = 'null' if existing_data is not None: address_encode['code'] = existing_address['code'] else: address_encode['code'] = addr_dict['code'] for input_tx in addr_dict['newInputTxs']: last_input_tx_index += 1 addresses_write_dict[addr_hash + '-i-' + str(last_input_tx_index)] = ( str(input_tx[0]) + '-' + str(input_tx[1]) + '-' + str(input_tx[2])) for output_tx in addr_dict['newOutputTxs']: last_output_tx_index += 1 addresses_write_dict[addr_hash + '-o-' + str(last_output_tx_index)] = ( str(output_tx[0]) + '-' + str(output_tx[1]) + '-' + str(output_tx[2])) for mined_hash in addr_dict['mined']: last_mined_block_index += 1 addresses_write_dict[addr_hash + '-b-' + str(last_mined_block_index)] = mined_hash address_encode['inputTxIndex'] = last_input_tx_index address_encode['outputTxIndex'] = last_output_tx_index address_encode['inputTokenTxIndex'] = last_input_token_tx_index address_encode['outputTokenTxIndex'] = last_output_token_tx_index address_encode['inputIntTxIndex'] = last_input_int_tx_index address_encode['outputIntTxIndex'] = last_output_int_tx_index address_encode['minedIndex'] = last_mined_block_index addresses_encode[addr_hash] = address_encode # Also add token information to the addresses. addresses_encode, updated_tokens, addresses_write_dict = self.fill_addrs_token_txs( addresses, addresses_encode, updated_tokens, addresses_write_dict) # Also add internal transactions to addresses addresses_encode, addresses_write_dict = self.fill_addrs_int_txs( addresses, addresses_encode, addresses_write_dict) return (addresses_encode, addresses_write_dict, updated_tokens, filtered_token_txs)
def get_blocks_by_datetime( self, limit: int, block_start: int, block_end: int) -> Union[List[Dict[str, Any]], None]: """ Retrieves multiple blocks based on specified datetime range. Args: limit: Maximum blocks to gather. block_start: Beginning datetime. block_end: End datetime. Returns: List of block dictionaries. """ block_indexes = [] # type: List[int] blocks = [] # type: List[Dict[str, Any]] retry_counter = 0 while True: try: it = self.db.iteritems() it.seek(b'timestamp-block-' + str(block_start).encode()) counter = 0 while True: data = it.get() timestamp = int(data[0].decode().split('-')[-1]) block_index = int(data[1].decode()) it.__next__() if timestamp < block_start: continue if timestamp > block_end: break block_indexes.append(block_index) counter += 1 if (counter >= limit and limit > 0): break break except rocksdb.errors.RocksIOError as e: if retry_counter >= 10: LOG.info('Too many failed retries. Stopping.') raise e if 'No such file or directory' in str(e): LOG.info('DB lookup failed. Retrying.') sleep(2) retry_counter += 1 if block_indexes == []: return None # Since DB is working with string-numbers things might be kind of tricky block_indexes.sort() for block_index in range(block_indexes[0], block_indexes[-1] + 1): raw_block = db_get_wrapper(self.db, b'block-' + str(block_index).encode()) block = coder.decode_block(raw_block) block['number'] = block_index transaction_hashes = block['transactions'].split('+') transactions = [] # type: List[Dict[str, Any]] if transaction_hashes == ['']: block['transactions'] = transactions blocks.append(block) continue for tx_hash in transaction_hashes: transaction = self.get_transaction_by_hash(tx_hash) transactions.append(transaction) # type: ignore block['transactions'] = transactions blocks.append(block) return blocks
def get_address(self, addr: str, time_from: int, time_to: int, val_from: int, val_to: int, no_tx_list: int) -> Union[List[Dict[str, Any]], None]: """ Get information of an address, with the possibility of filtering/limiting transactions. Args: addr: Ethereum address. time_from: Beginning datetime to take transactions from. time_to: Ending datetime to take transactions from. val_from: Minimum transferred currency of the transactions. val_to: Maximum transferred currency of transactions. no_tx_list: Maximum transactions to return. Returns: Address information along with its transactions. """ raw_address = db_get_wrapper(self.db, b'address-' + addr.encode()) if raw_address is None: return None address = coder.decode_address(raw_address) if address['code'] != '0x': raw_code = db_get_wrapper( self.db, b'address-contract-' + address['code'].encode()) address['code'] = raw_code.decode() input_transactions, output_transactions = ( self.get_transactions_of_address(addr, time_from, time_to, val_from, val_to, no_tx_list, True)) address.pop('inputTxIndex', None) address.pop('outputTxIndex', None) address['inputTransactions'] = input_transactions address['outputTransactions'] = output_transactions input_int_transactions, output_int_transactions = ( self.get_internal_txs_of_address(addr, time_from, time_to, val_from, val_to, no_tx_list, True)) address.pop('inputIntTxIndex', None) address.pop('outputIntTxIndex', None) address['inputInternalTransactions'] = input_int_transactions address['outputInternalTransactions'] = output_int_transactions mined_hashes = [] # type: List[bytes] if address['minedIndex'] > 0: prefix = 'associated-data-' + addr + '-b-' mined_hashes = db_iter_wrapper(self.db, prefix) address.pop('minedIndex', None) address['mined'] = list(map(lambda x: x.decode(), mined_hashes)) input_token_txs, output_token_txs = (self.get_token_txs_of_address( addr, time_from, time_to, no_tx_list, True)) address.pop('inputTokenTxIndex', None) address.pop('outputTokenTxIndex', None) address['inputTokenTransactions'] = input_token_txs address['outputTokenTransactions'] = output_token_txs return address
def get_token_txs_of_address(self, addr: str, time_from: int, time_to: int, no_tx_list: int, internal=False) -> Any: """ Get token txs of specified address, with filtering by time and transferred capital. Args: addr: Ethereum address. time_from: Beginning datetime to take transactions from. time_to: Ending datetime to take transactions from. no_tx_list: Maximum transactions to return. internal: Whether this method was called internally. Returns: List of token transactions of an address. """ raw_address = db_get_wrapper(self.db, b'address-' + addr.encode()) if raw_address is None: return None address = coder.decode_address(raw_address) input_token_txs = [] output_token_txs = [] input_token_tx_indexes = [] # type: List[bytes] output_token_tx_indexes = [] # type: List[bytes] if address['inputTokenTxIndex'] > 0: prefix = 'associated-data-' + addr + '-ti-' input_token_tx_indexes = db_iter_wrapper(self.db, prefix) if address['outputTokenTxIndex'] > 0: prefix = 'associated-data-' + addr + '-to-' output_token_tx_indexes = db_iter_wrapper(self.db, prefix) found_txs = 0 for token_tx_index in input_token_tx_indexes: if found_txs >= no_tx_list: break tx_decoded = token_tx_index.decode() tx_index, timestamp = tx_decoded.split('-') if (time_from <= int(timestamp) and time_to >= int(timestamp)): raw_tx = db_get_wrapper(self.db, b'token-tx-' + tx_index.encode()) token_tx = coder.decode_token_tx(raw_tx) input_token_txs.append(token_tx) found_txs += 1 for token_tx_index in output_token_tx_indexes: if found_txs >= no_tx_list: break tx_decoded = token_tx_index.decode() tx_index, timestamp = tx_decoded.split('-') if (time_from <= int(timestamp) and time_to >= int(timestamp)): raw_tx = db_get_wrapper(self.db, b'token-tx-' + tx_index.encode()) token_tx = coder.decode_token_tx(raw_tx) output_token_txs.append(token_tx) found_txs += 1 if internal: return (input_token_txs, output_token_txs) else: return input_token_txs + output_token_txs
def get_transactions_of_address(self, addr: str, time_from: int, time_to: int, val_from: int, val_to: int, no_tx_list: int, internal=False) -> Any: """ Get transactions of specified address, with filtering by time and transferred capital. Args: addr: Ethereum address. time_from: Beginning datetime to take transactions from. time_to: Ending datetime to take transactions from. val_from: Minimum transferred currency of the transactions. val_to: Maximum transferred currency of transactions. no_tx_list: Maximum transactions to return. internal: Whether this method was called internally. Returns: List of address transactions. """ raw_address = db_get_wrapper(self.db, b'address-' + addr.encode()) if raw_address is None: return None address = coder.decode_address(raw_address) input_tx_hashes = [] # type: List[bytes] output_tx_hashes = [] # type: List[bytes] if address['inputTxIndex'] > 0: prefix = 'associated-data-' + addr + '-i-' input_tx_hashes = db_iter_wrapper(self.db, prefix) if address['outputTxIndex'] > 0: prefix = 'associated-data-' + addr + '-o-' output_tx_hashes = db_iter_wrapper(self.db, prefix) found_txs = 0 input_transactions = [] for tx_data in input_tx_hashes: if found_txs >= no_tx_list: break tx_decoded = tx_data.decode() tx_hash, value, timestamp = tx_decoded.split('-') if (time_from <= int(timestamp) and time_to >= int(timestamp) and val_from <= int(value) and val_to >= int(value)): transaction = self.get_transaction_by_hash(tx_hash) transaction.pop('internalTransactions', None) # type: ignore input_transactions.append(transaction) found_txs += 1 output_transactions = [] for tx_data in output_tx_hashes: if found_txs >= no_tx_list: break tx_decoded = tx_data.decode() tx_hash, value, timestamp = tx_decoded.split('-') if (time_from <= int(timestamp) and time_to >= int(timestamp) and val_from <= int(value) and val_to >= int(value)): transaction = self.get_transaction_by_hash(tx_hash) transaction.pop('internalTransactions', None) # type: ignore output_transactions.append(transaction) found_txs += 1 if internal: return (input_transactions, output_transactions) else: return input_transactions + output_transactions