def get_json_net(self, url):
response = self.fetch_url(url)
try:
jsonObj = json.loads(response)
except ValueError as e:
#Something went wrong with JSON response from API, panic
msg = (("Expected JSON response from '%s' instead received '%s'") %
(url, str(response)))
logger.log_and_die(msg)
if jsonObj['found']:
self.consecutive_lookup_misses = 0
return jsonObj
else:
#Some addresses are not clustered due to limitations of the
# WalletExplorer.com API; for example, m-of-n escrow
# addresses.
self.consecutive_lookup_misses = (self.consecutive_lookup_misses +
1)
dprint("Not found for url: %s. %d consecutive misses so far." %
(url, self.consecutive_lookup_misses))
if (self.consecutive_lookup_misses ==
NUM_CONSECUTIVE_API_MISSES_TO_DIE):
msg = ("Error: Encountered %d consecutive misses to "
"WalletExplorer.com API. Please investigate and "
"increase the package constant "
"NUM_CONSECUTIVE_API_MISSES_TO_DIE if everything's "
"okay.") % NUM_CONSECUTIVE_API_MISSES_TO_DIE
logger.log_and_die(msg)
else:
raise custom_errors.NotFoundAtRemoteAPIError
def get_addresses_for_wallet_label_net(self, blame_label):
offset = 0
api_key = self.config.WALLETEXPLORER_API_KEY
urlbuilder = WalletExplorerURLBuilder()
url = urlbuilder.get_wallet_addresses_at_offset(blame_label, offset,
api_key)
json = self.get_json_net(url)
addresses = self.get_address_list_from_json(json)
addresses_count = 0
try:
addresses_count = json['addresses_count']
except KeyError as e:
logger.log_and_die(str(e))
#start by retrieving up to 100 addresses, continue until all fetched.
addresses_remaining = addresses_count - 100
offset = 100
while addresses_remaining > 0:
url = urlbuilder.get_wallet_addresses_at_offset(blame_label,
offset, api_key)
json = self.get_json_net(url)
addresses.extend(self.get_address_list_from_json(json))
addresses_remaining = addresses_remaining - 100
offset = offset + 100
return addresses
def get_tx_relayed_by_using_tx_id(self,
tx_id,
txObj=None,
benchmarker=None):
cached_relayed_by = self.database_connector.get_cached_relayed_by(
tx_id)
dprint("DB Cached relayed-by field for tx %s is: %s" %
(tx_id, str(cached_relayed_by)))
if cached_relayed_by is not None:
if benchmarker is not None:
benchmarker.increment_blockchain_info_queries_avoided_by_caching(
)
return cached_relayed_by
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_tx_info(tx_id)
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
return self.get_tx_relayed_by_using_txObj(jsonObj)
except ValueError as e:
#Something went wrong, panic
msg = ("Expected JSON response for tx id '%s', instead received "
"'%s'") % (str(tx_id), str(response))
logger.log_and_die(msg)
def get_addresses_for_wallet_label_net(self, blame_label):
offset = 0
api_key = self.config.WALLETEXPLORER_API_KEY
urlbuilder = WalletExplorerURLBuilder()
url = urlbuilder.get_wallet_addresses_at_offset(
blame_label, offset, api_key)
json = self.get_json_net(url)
addresses = self.get_address_list_from_json(json)
addresses_count = 0
try:
addresses_count = json['addresses_count']
except KeyError as e:
logger.log_and_die(str(e))
#start by retrieving up to 100 addresses, continue until all fetched.
addresses_remaining = addresses_count - 100
offset = 100
while addresses_remaining > 0:
url = urlbuilder.get_wallet_addresses_at_offset(
blame_label, offset, api_key)
json = self.get_json_net(url)
addresses.extend(self.get_address_list_from_json(json))
addresses_remaining = addresses_remaining - 100
offset = offset + 100
return addresses
def get_json_net(self, url):
response = self.fetch_url(url)
try:
jsonObj = json.loads(response)
except ValueError as e:
#Something went wrong with JSON response from API, panic
msg = (("Expected JSON response from '%s' instead received '%s'") %
(url, str(response)))
logger.log_and_die(msg)
if jsonObj['found']:
self.consecutive_lookup_misses = 0
return jsonObj
else:
#Some addresses are not clustered due to limitations of the
# WalletExplorer.com API; for example, m-of-n escrow
# addresses.
self.consecutive_lookup_misses = (
self.consecutive_lookup_misses + 1)
dprint("Not found for url: %s. %d consecutive misses so far." %
(url, self.consecutive_lookup_misses))
if (self.consecutive_lookup_misses ==
NUM_CONSECUTIVE_API_MISSES_TO_DIE):
msg = ("Error: Encountered %d consecutive misses to "
"WalletExplorer.com API. Please investigate and "
"increase the package constant "
"NUM_CONSECUTIVE_API_MISSES_TO_DIE if everything's "
"okay.") % NUM_CONSECUTIVE_API_MISSES_TO_DIE
logger.log_and_die(msg)
else:
raise custom_errors.NotFoundAtRemoteAPIError
def update_sendback_reuse_pct(self):
pct = ''
try:
pct = DECIMAL_FORMAT.format(100.0 * self.tx_sendback_reuse_num / self.tx_total_num)
except ZeroDivisionError:
logger.log_and_die("Tried to update sendback reuse % but got divby0 for block height " + str(self.block_num))
self.tx_sendback_reuse_pct = pct
print("DEBUG: Updated tx_sendback_reuse_pct for block %d is '%s'" % (self.block_num, self.tx_sendback_reuse_pct))
def update_receiver_histoy_pct(self):
pct = ''
try:
pct = DECIMAL_FORMAT.format(100.0 * self.tx_receiver_has_tx_history_num / self.tx_total_num)
except ZeroDivisionError:
logger.log_and_die("Tried to update receiver history % but got divby0 for block height " + str(self.block_num))
self.tx_receiver_has_tx_history_pct = pct
print("DEBUG: Updated tx_receiver_has_tx_history_pct for block %d is '%s'" % (self.block_num, self.tx_receiver_has_tx_history_pct))
def _get_input_address_list(self, tx_obj):
"""Helper for `process_tx`, gets input addr list from tx JSON."""
input_address_list = []
try:
for btc_input in tx_obj['inputs']:
if 'prev_out' in btc_input and 'addr' in btc_input['prev_out']:
input_address_list.append(btc_input['prev_out']['addr'])
return input_address_list
except KeyError as err:
logger.log_and_die("Missing element in tx_obj: '%s" % str(err))
def _get_input_address_list(self, tx_obj):
"""Helper for `process_tx`, gets input addr list from tx JSON."""
input_address_list = []
try:
for btc_input in tx_obj['inputs']:
if 'prev_out' in btc_input and 'addr' in btc_input['prev_out']:
input_address_list.append(btc_input['prev_out']['addr'])
return input_address_list
except KeyError as err:
logger.log_and_die("Missing element in tx_obj: '%s" % str(err))
def get_tx_relayed_by_using_txObj(self, txObj):
try:
relayed_by = txObj['relayed_by']
block_height = txObj['block_height']
tx_id = txObj['hash']
self.database_connector.record_relayed_by(tx_id, block_height,
relayed_by)
return relayed_by
except IndexError as e:
msg = ("relayed_by field missing from tx JSON object: %s" %
str(txObj))
logger.log_and_die(msg)
def add_history_reuse_blamed_party(self, blame_label,
num_tx_with_history_reuse):
if blame_label not in self.top_reuser_labels:
self.top_reuser_labels.append(blame_label)
if blame_label in self.party_label_to_pct_history_map:
logger.log_and_die(("Label '%s' has already been added as sender "
"to address with prior history to this stats "
"object for block height %d: %s") %
(blame_label, self.block_height, str(self)))
pct = DECIMAL_FORMAT.format(
(100.0 * num_tx_with_history_reuse / self.num_tx_total))
self.party_label_to_pct_history_map[blame_label] = pct
def add_sendback_reuse_blamed_party(self, blame_label,
num_tx_with_sendback_reuse):
if blame_label not in self.top_reuser_labels:
self.top_reuser_labels.append(blame_label)
if blame_label in self.party_label_to_pct_sendback_map:
logger.log_and_die(("Label '%s' has already been added as "
"send-back reuser to this stats object for "
"block height %d: %s") %
(blame_label, self.block_height, str(self)))
pct = DECIMAL_FORMAT.format(
100.0 * num_tx_with_sendback_reuse / self.num_tx_total)
self.party_label_to_pct_sendback_map[blame_label] = pct
def get_tx_relayed_by_using_txObj(self, txObj):
try:
relayed_by = txObj['relayed_by']
block_height = txObj['block_height']
tx_id = txObj['hash']
self.database_connector.record_relayed_by(tx_id, block_height,
relayed_by)
return relayed_by
except IndexError as e:
msg = ("relayed_by field missing from tx JSON object: %s" %
str(txObj))
logger.log_and_die(msg)
def add_history_reuse_blamed_party(self, blame_label,
num_tx_with_history_reuse):
if blame_label not in self.top_reuser_labels:
self.top_reuser_labels.append(blame_label)
if blame_label in self.party_label_to_pct_history_map:
logger.log_and_die(("Label '%s' has already been added as sender "
"to address with prior history to this stats "
"object for block height %d: %s") %
(blame_label, self.block_height, str(self)))
pct = DECIMAL_FORMAT.format(
(100.0 * num_tx_with_history_reuse / self.num_tx_total))
self.party_label_to_pct_history_map[blame_label] = pct
def add_sendback_reuse_blamed_party(self, blame_label,
num_tx_with_sendback_reuse):
if blame_label not in self.top_reuser_labels:
self.top_reuser_labels.append(blame_label)
if blame_label in self.party_label_to_pct_sendback_map:
logger.log_and_die(("Label '%s' has already been added as "
"send-back reuser to this stats object for "
"block height %d: %s") %
(blame_label, self.block_height, str(self)))
pct = DECIMAL_FORMAT.format(100.0 * num_tx_with_sendback_reuse /
self.num_tx_total)
self.party_label_to_pct_sendback_map[blame_label] = pct
def update_sendback_reuse_pct(self):
pct = ''
try:
pct = DECIMAL_FORMAT.format(100.0 * self.tx_sendback_reuse_num /
self.tx_total_num)
except ZeroDivisionError:
logger.log_and_die(
"Tried to update sendback reuse % but got divby0 for block height "
+ str(self.block_num))
self.tx_sendback_reuse_pct = pct
print("DEBUG: Updated tx_sendback_reuse_pct for block %d is '%s'" %
(self.block_num, self.tx_sendback_reuse_pct))
def is_first_transaction_for_address(self,
addr,
tx_id,
block_height,
benchmarker=None):
validate.check_address_and_die(addr, THIS_FILE)
#First, check the local database cache for this address. If it's not
# there, add it to the cache as an address that has been seen, and
# then do API lookups to determine whether this tx is the address's
# first.
if self.database_connector.has_address_been_seen_cache_if_not(
addr, block_height):
if benchmarker is not None:
benchmarker.increment_blockchain_info_queries_avoided_by_caching(
)
benchmarker.increment_blockchain_info_queries_avoided_by_caching(
)
return False
n_tx = self.get_number_of_transactions_for_address(addr)
offset = int(n_tx) - 1
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_tx_for_address_at_offset(addr, offset)
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
tx_list = jsonObj['txs']
if not tx_list:
#address has no history
return True
if tx_list[0]:
#Check if the first tx is the tx in question
first_tx_id = tx_list[0]['hash']
if first_tx_id == tx_id:
return True
else:
return False
else:
#Something wrong, this tx list seems to be neither empty nor
# contains a first tx, panic!
msg = ("Expected zero or one transactions for address '%s'" %
addr)
logger.log_and_die(msg)
except ValueError as e:
#Something weird came back from the API despite HTTP 200 OK, panic
msg = ("Expected a JSON response for address '%s', instead "
"received '%s'") % (addr, str(response))
logger.log_and_die(msg)
def update_receiver_histoy_pct(self):
pct = ''
try:
pct = DECIMAL_FORMAT.format(100.0 *
self.tx_receiver_has_tx_history_num /
self.tx_total_num)
except ZeroDivisionError:
logger.log_and_die(
"Tried to update receiver history % but got divby0 for block height "
+ str(self.block_num))
self.tx_receiver_has_tx_history_pct = pct
print(
"DEBUG: Updated tx_receiver_has_tx_history_pct for block %d is '%s'"
% (self.block_num, self.tx_receiver_has_tx_history_pct))
def is_first_transaction_for_address(self, addr, tx_id, block_height,
benchmarker = None):
validate.check_address_and_die(addr, THIS_FILE)
#First, check the local database cache for this address. If it's not
# there, add it to the cache as an address that has been seen, and
# then do API lookups to determine whether this tx is the address's
# first.
if self.database_connector.has_address_been_seen_cache_if_not(addr,
block_height):
if benchmarker is not None:
benchmarker.increment_blockchain_info_queries_avoided_by_caching()
benchmarker.increment_blockchain_info_queries_avoided_by_caching()
return False
n_tx = self.get_number_of_transactions_for_address(addr)
offset = int(n_tx) - 1
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_tx_for_address_at_offset(addr, offset)
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
tx_list = jsonObj['txs']
if not tx_list:
#address has no history
return True
if tx_list[0]:
#Check if the first tx is the tx in question
first_tx_id = tx_list[0]['hash']
if first_tx_id == tx_id:
return True
else:
return False
else:
#Something wrong, this tx list seems to be neither empty nor
# contains a first tx, panic!
msg = ("Expected zero or one transactions for address '%s'" %
addr)
logger.log_and_die(msg)
except ValueError as e:
#Something weird came back from the API despite HTTP 200 OK, panic
msg = ("Expected a JSON response for address '%s', instead "
"received '%s'") % (addr, str(response))
logger.log_and_die(msg)
def get_sender_label_from_json(self, remote_json):
if remote_json is None:
logger.log_and_die(("Called get_sender_label_from_json() with a "
"'remote_json' value of None."))
try:
if remote_json['is_coinbase']:
return None #There is no "sender" in a coinbase transaction, only a receiver.
elif remote_json['wallet_id'] is not None and \
remote_json['wallet_id']:
return remote_json['wallet_id']
else:
#WE.com has no label for this address, for some reason that I
# am not currently curious about :>
return None
except IndexError as e:
msg = (("One or more expected fields in the tx JSON object are "
"missing: '%s'. Exception: '%s'") %
(str(remote_json), str(e)))
logger.log_and_die(msg)
def get_sender_label_from_json(self, remote_json):
if remote_json is None:
logger.log_and_die(("Called get_sender_label_from_json() with a "
"'remote_json' value of None."))
try:
if remote_json['is_coinbase']:
return None #There is no "sender" in a coinbase transaction, only a receiver.
elif remote_json['wallet_id'] is not None and \
remote_json['wallet_id']:
return remote_json['wallet_id']
else:
#WE.com has no label for this address, for some reason that I
# am not currently curious about :>
return None
except IndexError as e:
msg = (("One or more expected fields in the tx JSON object are "
"missing: '%s'. Exception: '%s'") %
(str(remote_json), str(e)))
logger.log_and_die(msg)
def get_current_blockchain_block_height(self):
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_current_height_url()
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
height = jsonObj['height']
try:
height_as_int = int(height)
return height_as_int
except ValueError:
msg = "Invalid block height returned from API: %s" % str(height)
logger.log_and_die(msg)
except ValueError as e:
#Something weird came back from API despite HTTP 200 OK, panic
msg = ("Expected JSON response, instead received: '%s'" %
str(response))
logger.log_and_die(msg)
def get_current_blockchain_block_height(self):
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_current_height_url()
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
height = jsonObj['height']
try:
height_as_int = int(height)
return height_as_int
except ValueError:
msg = "Invalid block height returned from API: %s" % str(
height)
logger.log_and_die(msg)
except ValueError as e:
#Something weird came back from API despite HTTP 200 OK, panic
msg = ("Expected JSON response, instead received: '%s'" %
str(response))
logger.log_and_die(msg)
def get_tx_list(self, block_height, use_tx_out_addr_cache_only=False):
assert not use_tx_out_addr_cache_only #not applicable to remote reader
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_block_at_height_url(
block_height) #validates height
response = self.throttled_fetch_url(url)
current_num_retries = 0
while current_num_retries <= MAX_NUM_RETRIES:
try:
jsonObj = json.loads(response)
#BCI stores the main chain block in addition to orphaned blocks,
# so iterate through blocks at this height until we find the
# non-orphaned ("main chain") block.
for i in range(0, len(jsonObj['blocks'])):
if jsonObj['blocks'][i]['main_chain'] == True:
blockObj = jsonObj['blocks'][i]
tx_list = blockObj['tx']
return tx_list
msg = (("Could not find the main chain block in blocks listed "
"by remote API at block height %d") % block_height)
logger.log_and_die(msg)
except ValueError as e:
#Something weird came back from API despite HTTP 200 OK, try a
# few more times before giving up. For example, sometimes BCI
# API will return 'No Free Cluster Connection' when
# overloaded.
current_num_retries = current_num_retries + 1
#Exceeded maximum time we're willing to wait for the API, time to give
# up. Examples of irreconcilable return values: 'Unknown Error
# Fetching Blocks From Database' means we tried to grab a block at a
# height that doesn't exist yet.
msg = ("Expected JSON response for block at height %d, instead "
"received '%s'") % (block_height, str(response))
logger.log_and_die(msg)
def get_number_of_transactions_for_address(self, addr):
validate.check_address_and_die(addr, THIS_FILE)
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_number_of_transactions_for_address(addr)
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
n_tx = jsonObj['n_tx']
if not n_tx:
#panic
msg = ("Expected 'n_tx' from JSON response but found none: "
"'%s'") % (n_tx, str(response))
logger.log_and_die(msg)
try:
int(n_tx)
except ValueError:
msg = ("Expected integer value for 'n_tx' in JSON response but "
"found: '%s'") % (str(response))
logger.log_and_die(msg)
return n_tx
except ValueError as e:
#Something weird came back from the API despite HTTP 200 OK, panic
msg = ("Expected a JSON response for address '%s', instead "
"received '%s'") % (addr, str(response))
logger.log_and_die(msg)
def get_number_of_transactions_for_address(self, addr):
validate.check_address_and_die(addr, THIS_FILE)
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_number_of_transactions_for_address(addr)
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
n_tx = jsonObj['n_tx']
if not n_tx:
#panic
msg = ("Expected 'n_tx' from JSON response but found none: "
"'%s'") % (n_tx, str(response))
logger.log_and_die(msg)
try:
int(n_tx)
except ValueError:
msg = (
"Expected integer value for 'n_tx' in JSON response but "
"found: '%s'") % (str(response))
logger.log_and_die(msg)
return n_tx
except ValueError as e:
#Something weird came back from the API despite HTTP 200 OK, panic
msg = ("Expected a JSON response for address '%s', instead "
"received '%s'") % (addr, str(response))
logger.log_and_die(msg)
def get_output_address(self, tx_id, output_index, tx_json = None):
if USE_TX_OUTPUT_ADDR_CACHE_FIRST:
addr = self.database_connector.get_output_address(tx_id,
output_index)
if addr is not None:
return addr
#not in cache, fall back to querying RPC interface
if tx_json is None:
tx_json = self.get_decoded_tx(tx_id)
if 'vout' in tx_json and len(tx_json['vout']) > output_index and \
'scriptPubKey' in tx_json['vout'][output_index]:
if 'addresses' not in tx_json['vout'][output_index]['scriptPubKey']:
raise custom_errors.PrevOutAddressCannotBeDecodedError
else:
return tx_json['vout'][output_index]['scriptPubKey'][
'addresses'][0]
else:
msg = ("Missing element for vout in get_output_address() with tx "
"id %s and output index %d") % (tx_id, output_index)
logger.log_and_die(msg)
def get_tx_list(self, block_height, use_tx_out_addr_cache_only = False):
assert not use_tx_out_addr_cache_only #not applicable to remote reader
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_block_at_height_url(block_height) #validates height
response = self.throttled_fetch_url(url)
current_num_retries = 0
while current_num_retries <= MAX_NUM_RETRIES:
try:
jsonObj = json.loads(response)
#BCI stores the main chain block in addition to orphaned blocks,
# so iterate through blocks at this height until we find the
# non-orphaned ("main chain") block.
for i in range(0, len(jsonObj['blocks'])):
if jsonObj['blocks'][i]['main_chain'] == True:
blockObj = jsonObj['blocks'][i]
tx_list = blockObj['tx']
return tx_list
msg = (("Could not find the main chain block in blocks listed "
"by remote API at block height %d") % block_height)
logger.log_and_die(msg)
except ValueError as e:
#Something weird came back from API despite HTTP 200 OK, try a
# few more times before giving up. For example, sometimes BCI
# API will return 'No Free Cluster Connection' when
# overloaded.
current_num_retries = current_num_retries + 1
#Exceeded maximum time we're willing to wait for the API, time to give
# up. Examples of irreconcilable return values: 'Unknown Error
# Fetching Blocks From Database' means we tried to grab a block at a
# height that doesn't exist yet.
msg = ("Expected JSON response for block at height %d, instead "
"received '%s'") % (block_height, str(response))
logger.log_and_die(msg)
def get_output_address(self, tx_id, output_index, tx_json=None):
if USE_TX_OUTPUT_ADDR_CACHE_FIRST:
addr = self.database_connector.get_output_address(
tx_id, output_index)
if addr is not None:
return addr
#not in cache, fall back to querying RPC interface
if tx_json is None:
tx_json = self.get_decoded_tx(tx_id)
if 'vout' in tx_json and len(tx_json['vout']) > output_index and \
'scriptPubKey' in tx_json['vout'][output_index]:
if 'addresses' not in tx_json['vout'][output_index][
'scriptPubKey']:
raise custom_errors.PrevOutAddressCannotBeDecodedError
else:
return tx_json['vout'][output_index]['scriptPubKey'][
'addresses'][0]
else:
msg = ("Missing element for vout in get_output_address() with tx "
"id %s and output index %d") % (tx_id, output_index)
logger.log_and_die(msg)
def get_tx_relayed_by_using_tx_id(self, tx_id, txObj = None,
benchmarker = None):
cached_relayed_by = self.database_connector.get_cached_relayed_by(tx_id)
dprint("DB Cached relayed-by field for tx %s is: %s" %
(tx_id, str(cached_relayed_by)))
if cached_relayed_by is not None:
if benchmarker is not None:
benchmarker.increment_blockchain_info_queries_avoided_by_caching()
return cached_relayed_by
urlbuilder = BlockchainInfoURLBuilder(
self.config.BLOCKCHAIN_INFO_API_KEY)
url = urlbuilder.get_tx_info(tx_id)
response = self.throttled_fetch_url(url)
try:
jsonObj = json.loads(response)
return self.get_tx_relayed_by_using_txObj(jsonObj)
except ValueError as e:
#Something went wrong, panic
msg = ("Expected JSON response for tx id '%s', instead received "
"'%s'") % (str(tx_id), str(response))
logger.log_and_die(msg)
def fetch_url(url):
"""Fetch contents of remote page as string for specified url."""
current_retry_time_in_sec = 0
dprint("Fetching url: %s" % url)
response = ''
while current_retry_time_in_sec <= MAX_RETRY_TIME_IN_SEC:
if current_retry_time_in_sec:
sleep(current_retry_time_in_sec)
try:
response = urllib2.urlopen(url=url, timeout=NUM_SEC_TIMEOUT).read()
if response is None:
#For some reason, no handler handled the request
logger.log_and_die("No URL handler utilized.")
return response
except (urllib2.HTTPError, ssl.SSLError) as err:
#There was a problem fetching the page, maybe something other than
# HTTP 200 OK.
if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC:
logger.log_and_die(("Could not fetch url '%s': Code is %s "
"reason is '%s' full response: '%s'") %
(url, err.code, err.reason, response))
else:
current_retry_time_in_sec = current_retry_time_in_sec + 1
print(("Encountered HTTPError fetching '%s'. Will wait for "
"%d seconds before retrying. Error was: '%s'") %
(url, current_retry_time_in_sec, str(err)))
except urllib2.URLError as err:
if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC:
logger.log_and_die(("Invalid URL '%s' could not be fetched: "
"%s") % (url, str(err)))
else:
current_retry_time_in_sec = current_retry_time_in_sec + 1
print(("Encountered URLError fetching '%s'. Will wait for "
"%d seconds before retrying. Error was: '%s'") %
(url, current_retry_time_in_sec, str(err)))
except socket.error as err:
if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC:
logger.log_and_die(("URL '%s' could not be fetched (socket "
"error): %s") % (url, str(err)))
else:
current_retry_time_in_sec = current_retry_time_in_sec + 1
print(("Encountered socket error fetching '%s'. Will wait for "
"%d seconds before retrying. Error was: '%s'") %
(url, current_retry_time_in_sec, str(err)))
def fetch_url(url):
"""Fetch contents of remote page as string for specified url."""
current_retry_time_in_sec = 0
dprint("Fetching url: %s" % url)
response = ''
while current_retry_time_in_sec <= MAX_RETRY_TIME_IN_SEC:
if current_retry_time_in_sec:
sleep(current_retry_time_in_sec)
try:
response = urllib2.urlopen(url=url, timeout=NUM_SEC_TIMEOUT).read()
if response is None:
#For some reason, no handler handled the request
logger.log_and_die("No URL handler utilized.")
return response
except (urllib2.HTTPError, ssl.SSLError) as err:
#There was a problem fetching the page, maybe something other than
# HTTP 200 OK.
if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC:
logger.log_and_die(("Could not fetch url '%s': Code is %s "
"reason is '%s' full response: '%s'") %
(url, err.code, err.reason, response))
else:
current_retry_time_in_sec = current_retry_time_in_sec + 1
print(("Encountered HTTPError fetching '%s'. Will wait for "
"%d seconds before retrying. Error was: '%s'") %
(url, current_retry_time_in_sec, str(err)))
except urllib2.URLError as err:
if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC:
logger.log_and_die(("Invalid URL '%s' could not be fetched: "
"%s") % (url, str(err)))
else:
current_retry_time_in_sec = current_retry_time_in_sec + 1
print(("Encountered URLError fetching '%s'. Will wait for "
"%d seconds before retrying. Error was: '%s'") %
(url, current_retry_time_in_sec, str(err)))
except socket.error as err:
if current_retry_time_in_sec == MAX_RETRY_TIME_IN_SEC:
logger.log_and_die(("URL '%s' could not be fetched (socket "
"error): %s") % (url, str(err)))
else:
current_retry_time_in_sec = current_retry_time_in_sec + 1
print(("Encountered socket error fetching '%s'. Will wait for "
"%d seconds before retrying. Error was: '%s'") %
(url, current_retry_time_in_sec, str(err)))
def get_receiver_label_from_json(self, remote_json, receiver_address):
if remote_json is None:
logger.log_and_die(("Called get_receiver_label_from_json() with a "
"'remote_json' value of None."))
#find the address in the outputs
receiver = ''
try:
for btc_output in remote_json['out']:
out_addr = btc_output['address']
if out_addr == receiver_address:
return btc_output['wallet_id']
if not receiver:
#didn't find a matching address, panic
msg = ("Looked for addr '%s' in outputs but couldn't find the "
"matching output") % receiver_address
logger.log_and_die(msg)
except IndexError as e:
msg = (("One or more expected fields in the tx JSON object are "
"missing: '%s'. Exception: '%s'") %
(str(remote_json), str(e)))
logger.log_and_die(msg)
def get_receiver_label_from_json(self, remote_json, receiver_address):
if remote_json is None:
logger.log_and_die(("Called get_receiver_label_from_json() with a "
"'remote_json' value of None."))
#find the address in the outputs
receiver = ''
try:
for btc_output in remote_json['out']:
out_addr = btc_output['address']
if out_addr == receiver_address:
return btc_output['wallet_id']
if not receiver:
#didn't find a matching address, panic
msg = ("Looked for addr '%s' in outputs but couldn't find the "
"matching output") % receiver_address
logger.log_and_die(msg)
except IndexError as e:
msg = (("One or more expected fields in the tx JSON object are "
"missing: '%s'. Exception: '%s'") %
(str(remote_json), str(e)))
logger.log_and_die(msg)
