def get_json_net(self, url): response = self.fetch_url(url) try: jsonObj = json.loads(response) except ValueError as e: #Something went wrong with JSON response from API, panic msg = (("Expected JSON response from '%s' instead received '%s'") % (url, str(response))) logger.log_and_die(msg) if jsonObj['found']: self.consecutive_lookup_misses = 0 return jsonObj else: #Some addresses are not clustered due to limitations of the # WalletExplorer.com API; for example, m-of-n escrow # addresses. self.consecutive_lookup_misses = (self.consecutive_lookup_misses + 1) dprint("Not found for url: %s. %d consecutive misses so far." % (url, self.consecutive_lookup_misses)) if (self.consecutive_lookup_misses == NUM_CONSECUTIVE_API_MISSES_TO_DIE): msg = ("Error: Encountered %d consecutive misses to " "WalletExplorer.com API. Please investigate and " "increase the package constant " "NUM_CONSECUTIVE_API_MISSES_TO_DIE if everything's " "okay.") % NUM_CONSECUTIVE_API_MISSES_TO_DIE logger.log_and_die(msg) else: raise custom_errors.NotFoundAtRemoteAPIError
def get_addresses_for_wallet_label_net(self, blame_label): offset = 0 api_key = self.config.WALLETEXPLORER_API_KEY urlbuilder = WalletExplorerURLBuilder() url = urlbuilder.get_wallet_addresses_at_offset(blame_label, offset, api_key) json = self.get_json_net(url) addresses = self.get_address_list_from_json(json) addresses_count = 0 try: addresses_count = json['addresses_count'] except KeyError as e: logger.log_and_die(str(e)) #start by retrieving up to 100 addresses, continue until all fetched. addresses_remaining = addresses_count - 100 offset = 100 while addresses_remaining > 0: url = urlbuilder.get_wallet_addresses_at_offset(blame_label, offset, api_key) json = self.get_json_net(url) addresses.extend(self.get_address_list_from_json(json)) addresses_remaining = addresses_remaining - 100 offset = offset + 100 return addresses
def get_tx_relayed_by_using_tx_id(self, tx_id, txObj=None, benchmarker=None): cached_relayed_by = self.database_connector.get_cached_relayed_by( tx_id) dprint("DB Cached relayed-by field for tx %s is: %s" % (tx_id, str(cached_relayed_by))) if cached_relayed_by is not None: if benchmarker is not None: benchmarker.increment_blockchain_info_queries_avoided_by_caching( ) return cached_relayed_by urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_tx_info(tx_id) response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) return self.get_tx_relayed_by_using_txObj(jsonObj) except ValueError as e: #Something went wrong, panic msg = ("Expected JSON response for tx id '%s', instead received " "'%s'") % (str(tx_id), str(response)) logger.log_and_die(msg)
def get_addresses_for_wallet_label_net(self, blame_label): offset = 0 api_key = self.config.WALLETEXPLORER_API_KEY urlbuilder = WalletExplorerURLBuilder() url = urlbuilder.get_wallet_addresses_at_offset( blame_label, offset, api_key) json = self.get_json_net(url) addresses = self.get_address_list_from_json(json) addresses_count = 0 try: addresses_count = json['addresses_count'] except KeyError as e: logger.log_and_die(str(e)) #start by retrieving up to 100 addresses, continue until all fetched. addresses_remaining = addresses_count - 100 offset = 100 while addresses_remaining > 0: url = urlbuilder.get_wallet_addresses_at_offset( blame_label, offset, api_key) json = self.get_json_net(url) addresses.extend(self.get_address_list_from_json(json)) addresses_remaining = addresses_remaining - 100 offset = offset + 100 return addresses
def get_json_net(self, url): response = self.fetch_url(url) try: jsonObj = json.loads(response) except ValueError as e: #Something went wrong with JSON response from API, panic msg = (("Expected JSON response from '%s' instead received '%s'") % (url, str(response))) logger.log_and_die(msg) if jsonObj['found']: self.consecutive_lookup_misses = 0 return jsonObj else: #Some addresses are not clustered due to limitations of the # WalletExplorer.com API; for example, m-of-n escrow # addresses. self.consecutive_lookup_misses = ( self.consecutive_lookup_misses + 1) dprint("Not found for url: %s. %d consecutive misses so far." % (url, self.consecutive_lookup_misses)) if (self.consecutive_lookup_misses == NUM_CONSECUTIVE_API_MISSES_TO_DIE): msg = ("Error: Encountered %d consecutive misses to " "WalletExplorer.com API. Please investigate and " "increase the package constant " "NUM_CONSECUTIVE_API_MISSES_TO_DIE if everything's " "okay.") % NUM_CONSECUTIVE_API_MISSES_TO_DIE logger.log_and_die(msg) else: raise custom_errors.NotFoundAtRemoteAPIError
def update_sendback_reuse_pct(self): pct = '' try: pct = DECIMAL_FORMAT.format(100.0 * self.tx_sendback_reuse_num / self.tx_total_num) except ZeroDivisionError: logger.log_and_die("Tried to update sendback reuse % but got divby0 for block height " + str(self.block_num)) self.tx_sendback_reuse_pct = pct print("DEBUG: Updated tx_sendback_reuse_pct for block %d is '%s'" % (self.block_num, self.tx_sendback_reuse_pct))
def update_receiver_histoy_pct(self): pct = '' try: pct = DECIMAL_FORMAT.format(100.0 * self.tx_receiver_has_tx_history_num / self.tx_total_num) except ZeroDivisionError: logger.log_and_die("Tried to update receiver history % but got divby0 for block height " + str(self.block_num)) self.tx_receiver_has_tx_history_pct = pct print("DEBUG: Updated tx_receiver_has_tx_history_pct for block %d is '%s'" % (self.block_num, self.tx_receiver_has_tx_history_pct))
def _get_input_address_list(self, tx_obj): """Helper for `process_tx`, gets input addr list from tx JSON.""" input_address_list = [] try: for btc_input in tx_obj['inputs']: if 'prev_out' in btc_input and 'addr' in btc_input['prev_out']: input_address_list.append(btc_input['prev_out']['addr']) return input_address_list except KeyError as err: logger.log_and_die("Missing element in tx_obj: '%s" % str(err))
def get_tx_relayed_by_using_txObj(self, txObj): try: relayed_by = txObj['relayed_by'] block_height = txObj['block_height'] tx_id = txObj['hash'] self.database_connector.record_relayed_by(tx_id, block_height, relayed_by) return relayed_by except IndexError as e: msg = ("relayed_by field missing from tx JSON object: %s" % str(txObj)) logger.log_and_die(msg)
def add_history_reuse_blamed_party(self, blame_label, num_tx_with_history_reuse): if blame_label not in self.top_reuser_labels: self.top_reuser_labels.append(blame_label) if blame_label in self.party_label_to_pct_history_map: logger.log_and_die(("Label '%s' has already been added as sender " "to address with prior history to this stats " "object for block height %d: %s") % (blame_label, self.block_height, str(self))) pct = DECIMAL_FORMAT.format( (100.0 * num_tx_with_history_reuse / self.num_tx_total)) self.party_label_to_pct_history_map[blame_label] = pct
def add_sendback_reuse_blamed_party(self, blame_label, num_tx_with_sendback_reuse): if blame_label not in self.top_reuser_labels: self.top_reuser_labels.append(blame_label) if blame_label in self.party_label_to_pct_sendback_map: logger.log_and_die(("Label '%s' has already been added as " "send-back reuser to this stats object for " "block height %d: %s") % (blame_label, self.block_height, str(self))) pct = DECIMAL_FORMAT.format( 100.0 * num_tx_with_sendback_reuse / self.num_tx_total) self.party_label_to_pct_sendback_map[blame_label] = pct
def add_sendback_reuse_blamed_party(self, blame_label, num_tx_with_sendback_reuse): if blame_label not in self.top_reuser_labels: self.top_reuser_labels.append(blame_label) if blame_label in self.party_label_to_pct_sendback_map: logger.log_and_die(("Label '%s' has already been added as " "send-back reuser to this stats object for " "block height %d: %s") % (blame_label, self.block_height, str(self))) pct = DECIMAL_FORMAT.format(100.0 * num_tx_with_sendback_reuse / self.num_tx_total) self.party_label_to_pct_sendback_map[blame_label] = pct
def update_sendback_reuse_pct(self): pct = '' try: pct = DECIMAL_FORMAT.format(100.0 * self.tx_sendback_reuse_num / self.tx_total_num) except ZeroDivisionError: logger.log_and_die( "Tried to update sendback reuse % but got divby0 for block height " + str(self.block_num)) self.tx_sendback_reuse_pct = pct print("DEBUG: Updated tx_sendback_reuse_pct for block %d is '%s'" % (self.block_num, self.tx_sendback_reuse_pct))
def is_first_transaction_for_address(self, addr, tx_id, block_height, benchmarker=None): validate.check_address_and_die(addr, THIS_FILE) #First, check the local database cache for this address. If it's not # there, add it to the cache as an address that has been seen, and # then do API lookups to determine whether this tx is the address's # first. if self.database_connector.has_address_been_seen_cache_if_not( addr, block_height): if benchmarker is not None: benchmarker.increment_blockchain_info_queries_avoided_by_caching( ) benchmarker.increment_blockchain_info_queries_avoided_by_caching( ) return False n_tx = self.get_number_of_transactions_for_address(addr) offset = int(n_tx) - 1 urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_tx_for_address_at_offset(addr, offset) response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) tx_list = jsonObj['txs'] if not tx_list: #address has no history return True if tx_list[0]: #Check if the first tx is the tx in question first_tx_id = tx_list[0]['hash'] if first_tx_id == tx_id: return True else: return False else: #Something wrong, this tx list seems to be neither empty nor # contains a first tx, panic! msg = ("Expected zero or one transactions for address '%s'" % addr) logger.log_and_die(msg) except ValueError as e: #Something weird came back from the API despite HTTP 200 OK, panic msg = ("Expected a JSON response for address '%s', instead " "received '%s'") % (addr, str(response)) logger.log_and_die(msg)
def update_receiver_histoy_pct(self): pct = '' try: pct = DECIMAL_FORMAT.format(100.0 * self.tx_receiver_has_tx_history_num / self.tx_total_num) except ZeroDivisionError: logger.log_and_die( "Tried to update receiver history % but got divby0 for block height " + str(self.block_num)) self.tx_receiver_has_tx_history_pct = pct print( "DEBUG: Updated tx_receiver_has_tx_history_pct for block %d is '%s'" % (self.block_num, self.tx_receiver_has_tx_history_pct))
def is_first_transaction_for_address(self, addr, tx_id, block_height, benchmarker = None): validate.check_address_and_die(addr, THIS_FILE) #First, check the local database cache for this address. If it's not # there, add it to the cache as an address that has been seen, and # then do API lookups to determine whether this tx is the address's # first. if self.database_connector.has_address_been_seen_cache_if_not(addr, block_height): if benchmarker is not None: benchmarker.increment_blockchain_info_queries_avoided_by_caching() benchmarker.increment_blockchain_info_queries_avoided_by_caching() return False n_tx = self.get_number_of_transactions_for_address(addr) offset = int(n_tx) - 1 urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_tx_for_address_at_offset(addr, offset) response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) tx_list = jsonObj['txs'] if not tx_list: #address has no history return True if tx_list[0]: #Check if the first tx is the tx in question first_tx_id = tx_list[0]['hash'] if first_tx_id == tx_id: return True else: return False else: #Something wrong, this tx list seems to be neither empty nor # contains a first tx, panic! msg = ("Expected zero or one transactions for address '%s'" % addr) logger.log_and_die(msg) except ValueError as e: #Something weird came back from the API despite HTTP 200 OK, panic msg = ("Expected a JSON response for address '%s', instead " "received '%s'") % (addr, str(response)) logger.log_and_die(msg)
def get_sender_label_from_json(self, remote_json): if remote_json is None: logger.log_and_die(("Called get_sender_label_from_json() with a " "'remote_json' value of None.")) try: if remote_json['is_coinbase']: return None #There is no "sender" in a coinbase transaction, only a receiver. elif remote_json['wallet_id'] is not None and \ remote_json['wallet_id']: return remote_json['wallet_id'] else: #WE.com has no label for this address, for some reason that I # am not currently curious about :> return None except IndexError as e: msg = (("One or more expected fields in the tx JSON object are " "missing: '%s'. Exception: '%s'") % (str(remote_json), str(e))) logger.log_and_die(msg)
def get_current_blockchain_block_height(self): urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_current_height_url() response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) height = jsonObj['height'] try: height_as_int = int(height) return height_as_int except ValueError: msg = "Invalid block height returned from API: %s" % str(height) logger.log_and_die(msg) except ValueError as e: #Something weird came back from API despite HTTP 200 OK, panic msg = ("Expected JSON response, instead received: '%s'" % str(response)) logger.log_and_die(msg)
def get_current_blockchain_block_height(self): urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_current_height_url() response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) height = jsonObj['height'] try: height_as_int = int(height) return height_as_int except ValueError: msg = "Invalid block height returned from API: %s" % str( height) logger.log_and_die(msg) except ValueError as e: #Something weird came back from API despite HTTP 200 OK, panic msg = ("Expected JSON response, instead received: '%s'" % str(response)) logger.log_and_die(msg)
def get_tx_list(self, block_height, use_tx_out_addr_cache_only=False): assert not use_tx_out_addr_cache_only #not applicable to remote reader urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_block_at_height_url( block_height) #validates height response = self.throttled_fetch_url(url) current_num_retries = 0 while current_num_retries <= MAX_NUM_RETRIES: try: jsonObj = json.loads(response) #BCI stores the main chain block in addition to orphaned blocks, # so iterate through blocks at this height until we find the # non-orphaned ("main chain") block. for i in range(0, len(jsonObj['blocks'])): if jsonObj['blocks'][i]['main_chain'] == True: blockObj = jsonObj['blocks'][i] tx_list = blockObj['tx'] return tx_list msg = (("Could not find the main chain block in blocks listed " "by remote API at block height %d") % block_height) logger.log_and_die(msg) except ValueError as e: #Something weird came back from API despite HTTP 200 OK, try a # few more times before giving up. For example, sometimes BCI # API will return 'No Free Cluster Connection' when # overloaded. current_num_retries = current_num_retries + 1 #Exceeded maximum time we're willing to wait for the API, time to give # up. Examples of irreconcilable return values: 'Unknown Error # Fetching Blocks From Database' means we tried to grab a block at a # height that doesn't exist yet. msg = ("Expected JSON response for block at height %d, instead " "received '%s'") % (block_height, str(response)) logger.log_and_die(msg)
def get_number_of_transactions_for_address(self, addr): validate.check_address_and_die(addr, THIS_FILE) urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_number_of_transactions_for_address(addr) response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) n_tx = jsonObj['n_tx'] if not n_tx: #panic msg = ("Expected 'n_tx' from JSON response but found none: " "'%s'") % (n_tx, str(response)) logger.log_and_die(msg) try: int(n_tx) except ValueError: msg = ("Expected integer value for 'n_tx' in JSON response but " "found: '%s'") % (str(response)) logger.log_and_die(msg) return n_tx except ValueError as e: #Something weird came back from the API despite HTTP 200 OK, panic msg = ("Expected a JSON response for address '%s', instead " "received '%s'") % (addr, str(response)) logger.log_and_die(msg)
def get_number_of_transactions_for_address(self, addr): validate.check_address_and_die(addr, THIS_FILE) urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_number_of_transactions_for_address(addr) response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) n_tx = jsonObj['n_tx'] if not n_tx: #panic msg = ("Expected 'n_tx' from JSON response but found none: " "'%s'") % (n_tx, str(response)) logger.log_and_die(msg) try: int(n_tx) except ValueError: msg = ( "Expected integer value for 'n_tx' in JSON response but " "found: '%s'") % (str(response)) logger.log_and_die(msg) return n_tx except ValueError as e: #Something weird came back from the API despite HTTP 200 OK, panic msg = ("Expected a JSON response for address '%s', instead " "received '%s'") % (addr, str(response)) logger.log_and_die(msg)
def get_output_address(self, tx_id, output_index, tx_json = None): if USE_TX_OUTPUT_ADDR_CACHE_FIRST: addr = self.database_connector.get_output_address(tx_id, output_index) if addr is not None: return addr #not in cache, fall back to querying RPC interface if tx_json is None: tx_json = self.get_decoded_tx(tx_id) if 'vout' in tx_json and len(tx_json['vout']) > output_index and \ 'scriptPubKey' in tx_json['vout'][output_index]: if 'addresses' not in tx_json['vout'][output_index]['scriptPubKey']: raise custom_errors.PrevOutAddressCannotBeDecodedError else: return tx_json['vout'][output_index]['scriptPubKey'][ 'addresses'][0] else: msg = ("Missing element for vout in get_output_address() with tx " "id %s and output index %d") % (tx_id, output_index) logger.log_and_die(msg)
def get_tx_list(self, block_height, use_tx_out_addr_cache_only = False): assert not use_tx_out_addr_cache_only #not applicable to remote reader urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_block_at_height_url(block_height) #validates height response = self.throttled_fetch_url(url) current_num_retries = 0 while current_num_retries <= MAX_NUM_RETRIES: try: jsonObj = json.loads(response) #BCI stores the main chain block in addition to orphaned blocks, # so iterate through blocks at this height until we find the # non-orphaned ("main chain") block. for i in range(0, len(jsonObj['blocks'])): if jsonObj['blocks'][i]['main_chain'] == True: blockObj = jsonObj['blocks'][i] tx_list = blockObj['tx'] return tx_list msg = (("Could not find the main chain block in blocks listed " "by remote API at block height %d") % block_height) logger.log_and_die(msg) except ValueError as e: #Something weird came back from API despite HTTP 200 OK, try a # few more times before giving up. For example, sometimes BCI # API will return 'No Free Cluster Connection' when # overloaded. current_num_retries = current_num_retries + 1 #Exceeded maximum time we're willing to wait for the API, time to give # up. Examples of irreconcilable return values: 'Unknown Error # Fetching Blocks From Database' means we tried to grab a block at a # height that doesn't exist yet. msg = ("Expected JSON response for block at height %d, instead " "received '%s'") % (block_height, str(response)) logger.log_and_die(msg)
def get_output_address(self, tx_id, output_index, tx_json=None): if USE_TX_OUTPUT_ADDR_CACHE_FIRST: addr = self.database_connector.get_output_address( tx_id, output_index) if addr is not None: return addr #not in cache, fall back to querying RPC interface if tx_json is None: tx_json = self.get_decoded_tx(tx_id) if 'vout' in tx_json and len(tx_json['vout']) > output_index and \ 'scriptPubKey' in tx_json['vout'][output_index]: if 'addresses' not in tx_json['vout'][output_index][ 'scriptPubKey']: raise custom_errors.PrevOutAddressCannotBeDecodedError else: return tx_json['vout'][output_index]['scriptPubKey'][ 'addresses'][0] else: msg = ("Missing element for vout in get_output_address() with tx " "id %s and output index %d") % (tx_id, output_index) logger.log_and_die(msg)
def get_tx_relayed_by_using_tx_id(self, tx_id, txObj = None, benchmarker = None): cached_relayed_by = self.database_connector.get_cached_relayed_by(tx_id) dprint("DB Cached relayed-by field for tx %s is: %s" % (tx_id, str(cached_relayed_by))) if cached_relayed_by is not None: if benchmarker is not None: benchmarker.increment_blockchain_info_queries_avoided_by_caching() return cached_relayed_by urlbuilder = BlockchainInfoURLBuilder( self.config.BLOCKCHAIN_INFO_API_KEY) url = urlbuilder.get_tx_info(tx_id) response = self.throttled_fetch_url(url) try: jsonObj = json.loads(response) return self.get_tx_relayed_by_using_txObj(jsonObj) except ValueError as e: #Something went wrong, panic msg = ("Expected JSON response for tx id '%s', instead received " "'%s'") % (str(tx_id), str(response)) logger.log_and_die(msg)
def fetch_url(url): """Fetch contents of remote page as string for specified url.""" current_retry_time_in_sec = 0 dprint("Fetching url: %s" % url) response = '' while current_retry_time_in_sec <= MAX_RETRY_TIME_IN_SEC: if current_retry_time_in_sec: sleep(current_retry_time_in_sec) try: response = urllib2.urlopen(url=url, timeout=NUM_SEC_TIMEOUT).read() if response is None: #For some reason, no handler handled the request logger.log_and_die("No URL handler utilized.") return response except (urllib2.HTTPError, ssl.SSLError) as err: #There was a problem fetching the page, maybe something other than # HTTP 200 OK. if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC: logger.log_and_die(("Could not fetch url '%s': Code is %s " "reason is '%s' full response: '%s'") % (url, err.code, err.reason, response)) else: current_retry_time_in_sec = current_retry_time_in_sec + 1 print(("Encountered HTTPError fetching '%s'. Will wait for " "%d seconds before retrying. Error was: '%s'") % (url, current_retry_time_in_sec, str(err))) except urllib2.URLError as err: if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC: logger.log_and_die(("Invalid URL '%s' could not be fetched: " "%s") % (url, str(err))) else: current_retry_time_in_sec = current_retry_time_in_sec + 1 print(("Encountered URLError fetching '%s'. Will wait for " "%d seconds before retrying. Error was: '%s'") % (url, current_retry_time_in_sec, str(err))) except socket.error as err: if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC: logger.log_and_die(("URL '%s' could not be fetched (socket " "error): %s") % (url, str(err))) else: current_retry_time_in_sec = current_retry_time_in_sec + 1 print(("Encountered socket error fetching '%s'. Will wait for " "%d seconds before retrying. Error was: '%s'") % (url, current_retry_time_in_sec, str(err)))
def get_receiver_label_from_json(self, remote_json, receiver_address): if remote_json is None: logger.log_and_die(("Called get_receiver_label_from_json() with a " "'remote_json' value of None.")) #find the address in the outputs receiver = '' try: for btc_output in remote_json['out']: out_addr = btc_output['address'] if out_addr == receiver_address: return btc_output['wallet_id'] if not receiver: #didn't find a matching address, panic msg = ("Looked for addr '%s' in outputs but couldn't find the " "matching output") % receiver_address logger.log_and_die(msg) except IndexError as e: msg = (("One or more expected fields in the tx JSON object are " "missing: '%s'. Exception: '%s'") % (str(remote_json), str(e))) logger.log_and_die(msg)