class Crawler402():
""" Crawl endpoints to check status.
Check server socket connection and query endpoints for
price and recipient address.
"""
def __init__(self, endpoint_list, log_file):
"""Set up logging & member vars"""
# configure logging
logging.basicConfig(level=logging.INFO,
filename=log_file,
filemode='a',
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M')
self.console = logging.StreamHandler()
self.console.setLevel(logging.INFO)
logging.getLogger('402-crawler').addHandler(self.console)
self.logger = logging.getLogger('402-crawler')
self.endpoint_list = endpoint_list
def check_endpoints(self):
"""Crawl 402 endpoints"""
# create 402 client
self.bitrequests = BitTransferRequests(Wallet(), Config().username)
# crawl endpoints, check headers
self.logger.info("\nCrawling machine-payable endpoints...")
for endpoint in self.endpoint_list:
# extract domain name
name = endpoint.split('/',1)[0].split('.',1)[1]
# get server ip
server_ip = socket.gethostbyname(name)
# self.logger.info("Checking {0} on port {1}".format(server_ip, port))
self.logger.info("Checking {}...".format(endpoint))
# configure socket module
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_state = sock.connect_ex((server_ip, 80))
sock.close()
if server_state == 0:
try:
self.logger.info("Server state: {} is up!".format(endpoint))
response = self.bitrequests.get_402_info('https://'+endpoint)
self.logger.info("Price: {}".format(response['price']))
self.logger.info("Address: {}".format(response['bitcoin-address']))
except Exception as e:
self.logger.info("Could not read 402 payment headers.")
else:
self.logger.info("Server state: {} is down!".format('https://'+endpoint))
self.logger.info("Timestamp: {}\n".format(datetime.datetime.now()))
def buy(config, resource, data, method, data_file, output_file,
payment_method, max_price, info_only):
"""Buy from any machine payable endpoint
Note: The two1lib _buy function does not support simply returning an object,
until then, include a local copy here
"""
# If resource is a URL string, then bypass seller search
if URL_REGEXP.match(resource):
target_url = resource
seller = target_url
elif resource in DEMOS:
target_url = TWO1_MERCHANT_HOST + DEMOS[resource]["path"]
data = json.dumps(data)
else:
raise NotImplementedError('Endpoint search is not implemented!')
# Change default HTTP method from "GET" to "POST", if we have data
if method == "GET" and (data or data_file):
method = "POST"
# Set default headers for making bitrequests with JSON-like data
headers = {'Content-Type': 'application/json'}
try:
# Find the correct payment method
if payment_method == 'offchain':
bit_req = BitTransferRequests(config.machine_auth, config.username)
elif payment_method == 'onchain':
bit_req = OnChainRequests(config.wallet)
else:
raise Exception('Payment method does not exist.')
# Make the request
if info_only:
res = bit_req.get_402_info(target_url)
else:
res = bit_req.request(
method.lower(), target_url, max_price=max_price,
data=data or data_file, headers=headers)
except ResourcePriceGreaterThanMaxPriceError as e:
config.log(UxString.Error.resource_price_greater_than_max_price.format(e))
return
except Exception as e:
if 'Insufficient funds.' in str(e):
config.log(UxString.Error.insufficient_funds_mine_more.format(
DEFAULT_ONCHAIN_BUY_FEE
))
else:
config.log(str(e), fg="red")
return
# Output results to user
if output_file:
# Write response output file
output_file.write(res.content)
elif info_only:
# Print headers that are related to 402 payment required
for key, val in res.items():
config.log('{}: {}'.format(key, val))
elif resource in DEMOS:
config.log(DEMOS[resource]["formatter"](res))
else:
response = res.json()
# Clean up names
for index, elem in enumerate(response):
if elem['name'] is None:
response[index]['name'] = 'Please name me'
elif len(elem['name']) == 0:
response[index]['name'] = 'Please name me'
else:
response[index]['name'] = response[index]['name'].title()
print(elem['description'])
if elem['description'] is None:
try:
response[index]['description'] = elem['owner'].title() + ' is a bad endpoint operator and forgot to place a description'
except:
response[index]['description'] = 'Anonymous is a bad endpoint operator and forgot to place a description'
# Any description greater than 66 characters causes the text to overflow, this enforces a limit
elif len(elem['description']) > 63:
response[index]['description'] = response[index]['description'][:63] + '...'
# Write response to console
return response
# Write the amount paid out if something was truly paid
if not info_only and hasattr(res, 'amount_paid'):
client = rest_client.TwentyOneRestClient(TWO1_HOST,
config.machine_auth,
config.username)
user_balances = _get_balances(config, client)
if payment_method == 'offchain':
balance_amount = user_balances.twentyone
balance_type = '21.co'
elif payment_method == 'onchain':
balance_amount = user_balances.onchain
balance_type = 'blockchain'
# Record the transaction if it was a payable request
if hasattr(res, 'paid_amount'):
config.log_purchase(s=seller,
r=resource,
p=res.paid_amount,
d=str(datetime.datetime.today()))
def buy(config, resource, data, method, data_file, output_file,
payment_method, max_price, info_only):
"""Buy from any machine payable endpoint
Note: The two1lib _buy function does not support simply returning an object,
until then, include a local copy here
"""
# If resource is a URL string, then bypass seller search
if URL_REGEXP.match(resource):
target_url = resource
seller = target_url
elif resource in DEMOS:
target_url = TWO1_MERCHANT_HOST + DEMOS[resource]["path"]
data = json.dumps(data)
else:
raise NotImplementedError('Endpoint search is not implemented!')
# Change default HTTP method from "GET" to "POST", if we have data
if method == "GET" and (data or data_file):
method = "POST"
# Set default headers for making bitrequests with JSON-like data
headers = {'Content-Type': 'application/json'}
try:
# Find the correct payment method
if payment_method == 'offchain':
bit_req = BitTransferRequests(config.machine_auth,
config.username)
elif payment_method == 'onchain':
bit_req = OnChainRequests(config.wallet)
else:
raise Exception('Payment method does not exist.')
# Make the request
if info_only:
res = bit_req.get_402_info(target_url)
else:
res = bit_req.request(method.lower(),
target_url,
max_price=max_price,
data=data or data_file,
headers=headers)
except ResourcePriceGreaterThanMaxPriceError as e:
config.log(
UxString.Error.resource_price_greater_than_max_price.format(e))
return
except Exception as e:
if 'Insufficient funds.' in str(e):
config.log(
UxString.Error.insufficient_funds_mine_more.format(
DEFAULT_ONCHAIN_BUY_FEE))
else:
config.log(str(e), fg="red")
return
# Output results to user
if output_file:
# Write response output file
output_file.write(res.content)
elif info_only:
# Print headers that are related to 402 payment required
for key, val in res.items():
config.log('{}: {}'.format(key, val))
elif resource in DEMOS:
config.log(DEMOS[resource]["formatter"](res))
else:
response = res.json()
# Clean up names
for index, elem in enumerate(response):
if elem['name'] is None:
response[index]['name'] = 'Please name me'
elif len(elem['name']) == 0:
response[index]['name'] = 'Please name me'
else:
response[index]['name'] = response[index]['name'].title()
print(elem['description'])
if elem['description'] is None:
try:
response[index]['description'] = elem['owner'].title(
) + ' is a bad endpoint operator and forgot to place a description'
except:
response[index][
'description'] = 'Anonymous is a bad endpoint operator and forgot to place a description'
# Any description greater than 66 characters causes the text to overflow, this enforces a limit
elif len(elem['description']) > 63:
response[index]['description'] = response[index][
'description'][:63] + '...'
# Write response to console
return response
# Write the amount paid out if something was truly paid
if not info_only and hasattr(res, 'amount_paid'):
client = rest_client.TwentyOneRestClient(TWO1_HOST,
config.machine_auth,
config.username)
user_balances = _get_balances(config, client)
if payment_method == 'offchain':
balance_amount = user_balances.twentyone
balance_type = '21.co'
elif payment_method == 'onchain':
balance_amount = user_balances.onchain
balance_type = 'blockchain'
# Record the transaction if it was a payable request
if hasattr(res, 'paid_amount'):
config.log_purchase(s=seller,
r=resource,
p=res.paid_amount,
d=str(datetime.datetime.today()))
def _buy(config, resource, data, method, data_file, output_file,
payment_method, max_price, info_only):
# If resource is a URL string, then bypass seller search
if URL_REGEXP.match(resource):
target_url = resource
seller = target_url
elif resource in DEMOS:
target_url = TWO1_MERCHANT_HOST + DEMOS[resource]["path"]
data = json.dumps(data)
else:
raise NotImplementedError('Endpoint search is not implemented!')
# Change default HTTP method from "GET" to "POST", if we have data
if method == "GET" and (data or data_file):
method = "POST"
# Set default headers for making bitrequests with JSON-like data
headers = {'Content-Type': 'application/json'}
try:
# Find the correct payment method
if payment_method == 'offchain':
bit_req = BitTransferRequests(config.machine_auth, config.username)
elif payment_method == 'onchain':
bit_req = OnChainRequests(config.wallet)
else:
raise Exception('Payment method does not exist.')
# Make the request
if info_only:
res = bit_req.get_402_info(target_url)
else:
res = bit_req.request(
method.lower(), target_url, max_price=max_price,
data=data or data_file, headers=headers)
except ResourcePriceGreaterThanMaxPriceError as e:
config.log(UxString.Error.resource_price_greater_than_max_price.format(e))
return
except Exception as e:
if 'Insufficient funds.' in str(e):
config.log(UxString.Error.insufficient_funds_mine_more.format(
DEFAULT_ONCHAIN_BUY_FEE
))
else:
config.log(str(e), fg="red")
return
# Output results to user
if output_file:
# Write response output file
output_file.write(res.content)
elif info_only:
# Print headers that are related to 402 payment required
for key, val in res.items():
config.log('{}: {}'.format(key, val))
elif resource in DEMOS:
config.log(DEMOS[resource]["formatter"](res))
else:
# Write response to console
config.log(res.text)
# Write the amount paid out
if not info_only:
client = rest_client.TwentyOneRestClient(TWO1_HOST,
config.machine_auth,
config.username)
twentyone_balance, balance_c, pending_transactions, flushed_earnings = \
_get_balances(config, client)
config.log("You spent: %s Satoshis. Remaining 21.co balance: %s Satoshis." % (res.amount_paid, twentyone_balance))
# Record the transaction if it was a payable request
if hasattr(res, 'paid_amount'):
config.log_purchase(s=seller,
r=resource,
p=res.paid_amount,
d=str(datetime.datetime.today()))
def _buy(config, resource, data, method, data_file, output_file,
payment_method, max_price, info_only):
# If resource is a URL string, then bypass seller search
if URL_REGEXP.match(resource):
target_url = resource
seller = target_url
elif resource in DEMOS:
target_url = TWO1_MERCHANT_HOST + DEMOS[resource]["path"]
data = json.dumps(data)
else:
raise NotImplementedError('Endpoint search is not implemented!')
# Change default HTTP method from "GET" to "POST", if we have data
if method == "GET" and (data or data_file):
method = "POST"
# Set default headers for making bitrequests with JSON-like data
headers = {'Content-Type': 'application/json'}
try:
# Find the correct payment method
if payment_method == 'offchain':
bit_req = BitTransferRequests(config.machine_auth, config.username)
elif payment_method == 'onchain':
bit_req = OnChainRequests(config.wallet)
elif payment_method == 'channel':
bit_req = ChannelRequests(config.wallet)
channel_list = bit_req._channelclient.list()
if not channel_list:
confirmed = click.confirm(UxString.buy_channel_warning.format(
bit_req.DEFAULT_DEPOSIT_AMOUNT,
PaymentChannelStateMachine.PAYMENT_TX_MIN_OUTPUT_AMOUNT), default=True)
if not confirmed:
raise Exception(UxString.buy_channel_aborted)
else:
raise Exception('Payment method does not exist.')
# Make the request
if info_only:
res = bit_req.get_402_info(target_url)
else:
res = bit_req.request(
method.lower(), target_url, max_price=max_price,
data=data or data_file, headers=headers)
except ResourcePriceGreaterThanMaxPriceError as e:
config.log(UxString.Error.resource_price_greater_than_max_price.format(e))
return
except Exception as e:
f = get_fees()
buy_fee = 2 * f['per_input'] + f['per_output']
if 'Insufficient funds.' in str(e):
config.log(UxString.Error.insufficient_funds_mine_more.format(
buy_fee
))
else:
config.log(str(e), fg="red")
return
# Output results to user
if output_file:
# Write response output file
output_file.write(res.content)
elif info_only:
# Print headers that are related to 402 payment required
for key, val in res.items():
config.log('{}: {}'.format(key, val))
elif resource in DEMOS:
config.log(DEMOS[resource]["formatter"](res))
else:
# Write response to console
config.log(res.text)
# Write the amount paid out if something was truly paid
if not info_only and hasattr(res, 'amount_paid'):
client = rest_client.TwentyOneRestClient(TWO1_HOST,
config.machine_auth,
config.username)
user_balances = _get_balances(config, client)
if payment_method == 'offchain':
balance_amount = user_balances.twentyone
balance_type = '21.co'
elif payment_method == 'onchain':
balance_amount = user_balances.onchain
balance_type = 'blockchain'
elif payment_method == 'channel':
balance_amount = user_balances.channels
balance_type = 'payment channels'
config.log("You spent: %s Satoshis. Remaining %s balance: %s Satoshis." % (
res.amount_paid, balance_type, balance_amount))
# Record the transaction if it was a payable request
if hasattr(res, 'paid_amount'):
config.log_purchase(s=seller,
r=resource,
p=res.paid_amount,
d=str(datetime.datetime.today()))
# Loading developer libraries
import json
from two1.commands.config import Config
from two1.lib.wallet import Wallet
from two1.lib.bitrequests import BitTransferRequests
wallet = Wallet()
username = Config().username
requests = BitTransferRequests(wallet, username)
# Set up the server to run locally
server_url = 'http://localhost:5000/'
# Request the 402 end-points from the server and assign price and address to variables.
info = requests.get_402_info(url=server_url+'top-stories')
endpoint_info = dict(info)
price = int(endpoint_info['price'])
address = str(endpoint_info['bitcoin-address'])
# Array of all the top-story sections
sections = ["home", "world", "national", "politics",
"nyregion", "business", "opinion", "technology",
"health", "sports", "arts", "fashion", "dining",
"travel", "magazine", "realestate"]
def get_top_stories(section):
params = {
'section': section
}
# Set up wallet proxy
wallet = Wallet()
# Read local user account data
config = Config()
username = Config().username
# Initialize rest client to communicate with 21.co API
client = rest_client.TwentyOneRestClient(TWO1_HOST, config.machine_auth, config.username)
# Set up bitrequests client using BriTransfer payment method for 402 requests
requests = BitTransferRequests(wallet, username)
# Check the spot price for our API call
response = requests.get_402_info(url=SERVER_URL)
endpoint_info = dict(response)
price = int(endpoint_info['price'])
def get_element(arr, prop, val):
for elem in arr:
if elem[prop] == val:
return val
time.sleep(1)
# fast bitcoin aware api NOx
def fast_get_element(arr, prop, val):
body = {
'array': json.dumps(arr),
'property': prop,
'value': val
class Crawler402():
""" Crawl endpoints to check status.
Check server socket connection and query endpoints for
price and recipient address.
"""
def __init__(self, endpoint_list, log_file):
"""Set up logging & member vars"""
# configure logging
logging.basicConfig(
level=logging.INFO,
filename=log_file,
filemode='a',
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M')
self.console = logging.StreamHandler()
self.console.setLevel(logging.INFO)
logging.getLogger('402-crawler').addHandler(self.console)
self.logger = logging.getLogger('402-crawler')
self.endpoint_list = endpoint_list
def check_endpoints(self):
"""Crawl 402 endpoints"""
# create 402 client
self.bitrequests = BitTransferRequests(Wallet(), Config().username)
# crawl endpoints, check headers
self.logger.info("\nCrawling machine-payable endpoints...")
for endpoint in self.endpoint_list:
# extract domain name
name = endpoint.split('/', 1)[0].split('.', 1)[1]
# get server ip
server_ip = socket.gethostbyname(name)
# self.logger.info("Checking {0} on port {1}".format(server_ip, port))
self.logger.info("Checking {}...".format(endpoint))
# configure socket module
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_state = sock.connect_ex((server_ip, 80))
sock.close()
if server_state == 0:
try:
self.logger.info(
"Server state: {} is up!".format(endpoint))
response = self.bitrequests.get_402_info('https://' +
endpoint)
self.logger.info("Price: {}".format(response['price']))
self.logger.info("Address: {}".format(
response['bitcoin-address']))
except Exception as e:
self.logger.info("Could not read 402 payment headers.")
else:
self.logger.info(
"Server state: {} is down!".format('https://' + endpoint))
self.logger.info("Timestamp: {}\n".format(datetime.datetime.now()))
# Loading developer libraries
import json
from two1.commands.config import Config
from two1.lib.wallet import Wallet
from two1.lib.bitrequests import BitTransferRequests
wallet = Wallet()
username = Config().username
requests = BitTransferRequests(wallet, username)
# Set up the server to run locally
server_url = 'http://localhost:5000/'
# Request the 402 end-points from the server and assign price and address to variables.
info = requests.get_402_info(url=server_url + 'top-stories')
endpoint_info = dict(info)
price = int(endpoint_info['price'])
address = str(endpoint_info['bitcoin-address'])
# Array of all the top-story sections
sections = [
"home", "world", "national", "politics", "nyregion", "business", "opinion",
"technology", "health", "sports", "arts", "fashion", "dining", "travel",
"magazine", "realestate"
]
def get_top_stories(section):
params = {'section': section}
response = requests.post(url=server_url + 'top-stories', data=params)
# set up wallet proxy
wallet = Wallet()
# read local user account data
config = Config()
username = Config().username
# initialize rest client to communicate with 21.co API
client = rest_client.TwentyOneRestClient(TWO1_HOST, config.machine_auth, config.username)
# set up bitrequests client using BitTransfer payment method for managing 402 requests
requests = BitTransferRequests(wallet, username)
# check the spot price for our API call
response = requests.get_402_info(url='http://localhost:5000/fastfind')
endpoint_info = dict(response)
price = int(endpoint_info['price'])
print(endpoint_info)
# sample data
data1 = [
{'height': 4},
{'height': 3},
{'height': 6},
{'height': 4},
{'height': 3},
{'height': 6},
{'height': 4},
{'height': 3},
{'height': 6},
def _buy(config, resource, data, method, data_file, output_file,
payment_method, max_price, info_only):
"""
Buys bitcoin payable content over http
Todo:
reduce number of input args
Exception is too general, raise a different exception when user cannot pay
Args:
config (Config): config object used for getting .two1 information
resource (str): resource or content to purchase
method (str): HTTP request method, defaults to GET
data_file (str): name of the data file to send in HTTP body
output_file (str): Output file name
payment_method (str): Type of payment used in the purchase: offchain, onchain, channel
max_price (int): Max price of resource
info_only (bool): Flag which will only get info and not purcahase the resource
Raises:
NotImplementedError: if endpoint or resource is not valid
ResourcePriceGreaterThanMaxPriceError: If the resource price is greater than the max price
"""
# If resource is a URL string, then bypass seller search
if URL_REGEXP.match(resource):
target_url = resource
seller = target_url
elif re.match(r'^(((\w*)(\/){0,1})(\w*)){0,2}(\/){0,1}$',
resource) and resource not in DEMOS:
target_url = 'https://mkt.21.co/' + resource
seller = target_url
elif resource in DEMOS:
target_url = TWO1_MERCHANT_HOST + DEMOS[resource]["path"]
data = json.dumps(data)
else:
# If we can't figure out the resource type, attempt to use `http`
target_url = 'http://' + resource
# Change default HTTP method from "GET" to "POST", if we have data
if method == "GET" and (data or data_file):
method = "POST"
# Set default headers for making bitrequests with JSON-like data
headers = {'Content-Type': 'application/json'}
try:
# Find the correct payment method
if payment_method == 'offchain':
bit_req = BitTransferRequests(config.machine_auth, config.username)
elif payment_method == 'onchain':
bit_req = OnChainRequests(config.wallet)
elif payment_method == 'channel':
bit_req = ChannelRequests(config.wallet)
channel_list = bit_req._channelclient.list()
if not channel_list:
confirmed = click.confirm(UxString.buy_channel_warning.format(
bit_req.DEFAULT_DEPOSIT_AMOUNT,
PaymentChannelStateMachine.PAYMENT_TX_MIN_OUTPUT_AMOUNT),
default=True)
if not confirmed:
raise Exception(UxString.buy_channel_aborted)
else:
raise Exception('Payment method does not exist.')
# Make the request
if info_only:
res = bit_req.get_402_info(target_url)
else:
res = bit_req.request(method.lower(),
target_url,
max_price=max_price,
data=data or data_file,
headers=headers)
except ResourcePriceGreaterThanMaxPriceError as e:
config.log(
UxString.Error.resource_price_greater_than_max_price.format(e))
return
except Exception as e:
f = get_fees()
buy_fee = 2 * f['per_input'] + f['per_output']
if 'Insufficient funds.' in str(e):
config.log(
UxString.Error.insufficient_funds_mine_more.format(buy_fee))
else:
config.log(str(e), fg="red")
return
# Output results to user
if output_file:
# Write response output file
output_file.write(res.content)
elif info_only:
# Print headers that are related to 402 payment required
for key, val in res.items():
config.log('{}: {}'.format(key, val))
elif resource in DEMOS:
config.log(DEMOS[resource]["formatter"](res))
else:
# Write response to console
config.log(res.text)
# Write the amount paid out if something was truly paid
if not info_only and hasattr(res, 'amount_paid'):
client = rest_client.TwentyOneRestClient(TWO1_HOST,
config.machine_auth,
config.username)
user_balances = _get_balances(config, client)
if payment_method == 'offchain':
balance_amount = user_balances.twentyone
balance_type = '21.co'
elif payment_method == 'onchain':
balance_amount = user_balances.onchain
balance_type = 'blockchain'
elif payment_method == 'channel':
balance_amount = user_balances.channels
balance_type = 'payment channels'
config.log(
"You spent: %s Satoshis. Remaining %s balance: %s Satoshis." %
(res.amount_paid, balance_type, balance_amount))
# Record the transaction if it was a payable request
if hasattr(res, 'paid_amount'):
config.log_purchase(s=seller,
r=resource,
p=res.paid_amount,
d=str(datetime.datetime.today()))
