def mergeAdders(newAdderList):
'''
Merge peer adders list to our database
'''
blacklist = onionrblacklist.OnionrBlackList()
retVal = False
if newAdderList != False:
for adder in newAdderList.split(','):
adder = adder.strip()
if not adder in keydb.listkeys.list_adders(
randomOrder=False) and not adder in gettransports.get(
) and not blacklist.inBlacklist(adder):
if keydb.addkeys.add_address(adder):
# Check if we have the maximum amount of allowed stored peers
if config.get('peers.max_stored_peers') > len(
keydb.listkeys.list_adders()):
logger.info('Added %s to db.' % adder, timestamp=True)
retVal = True
else:
logger.warn(
'Reached the maximum amount of peers in the net database as allowed by your config.'
)
else:
pass
#logger.debug('%s is either our address or already in our DB' % adder)
return retVal
def add_address(address):
"""Add an address to the address database (only tor currently)"""
if type(address) is None or len(address) == 0:
return False
if stringvalidators.validate_transport(address):
if address in gettransports.get():
return False
conn = sqlite3.connect(dbfiles.address_info_db,
timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# check if address is in database
# this is safe to do because the address is validated above, but we strip some chars here too just in case
address = address.replace('\'', '').replace(';', '').replace(
'"', '').replace('\\', '')
for i in c.execute("SELECT * FROM adders WHERE address = ?;",
(address, )):
try:
if i[0] == address:
conn.close()
return False
except ValueError:
pass
except IndexError:
pass
t = (address, 1)
c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
conn.commit()
conn.close()
return True
else:
return False
def validate_request():
"""Validate request has the correct hostname"""
# If high security level, deny requests to public
# (HS should be disabled anyway for Tor, but might not be for I2P)
g.is_onionr_client = False
transports = gettransports.get()
if public_api.config.get('general.security_level', default=1) > 0:
abort(403)
if request.host not in transports:
# Abort conn if wrong HTTP hostname, to prevent DNS rebinding
abort(403)
public_api.hitCount += 1 # raise hit count for valid requests
try:
if 'onionr' in request.headers['User-Agent'].lower():
g.is_onionr_client = True
else:
g.is_onionr_client = False
except KeyError:
g.is_onionr_client = False
# Add shared objects
try:
g.too_many = public_api._too_many
except KeyError:
g.too_many = None
def lookup_new_peer_transports_with_communicator(comm_inst):
logger.info('Looking up new addresses...')
tryAmount = 1
newPeers = []
transports = gettransports.get()
for i in range(tryAmount):
# Download new peer address list from random online peers
if len(newPeers) > 10000:
# Don't get new peers if we have too many queued up
break
peer = onlinepeers.pick_online_peer(comm_inst)
newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
try:
newPeers = newAdders.split(',')
except AttributeError:
pass
else:
# Validate new peers are good format and not already in queue
invalid = []
for x in newPeers:
x = x.strip()
if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports:
# avoid adding if its our address
invalid.append(x)
for x in invalid:
try:
newPeers.remove(x)
except ValueError:
pass
comm_inst.newPeers.extend(newPeers)
comm_inst.decrementThreadCount('lookup_new_peer_transports_with_communicator')
def announce_node(daemon):
'''Announce our node to our peers'''
ret_data = False
announce_fail = False
# Do not let announceCache get too large
if len(daemon.announceCache) >= 10000:
daemon.announceCache.popitem()
if daemon.config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in daemon.onlinePeers:
if not i in daemon.announceCache and not i in daemon.announceProgress:
peer = i
break
else:
peer = onlinepeers.pick_online_peer(daemon)
for x in range(1):
try:
ourID = gettransports.get()[0]
except IndexError:
break
url = 'http://' + peer + '/announce'
data = {'node': ourID}
combinedNodes = ourID + peer
if ourID != 1:
existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
# Reset existingRand if it no longer meets the minimum POW
if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
existingRand = ''
if peer in daemon.announceCache:
data['random'] = daemon.announceCache[peer]
elif len(existingRand) > 0:
data['random'] = existingRand
else:
daemon.announceProgress[peer] = True
proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
del daemon.announceProgress[peer]
try:
data['random'] = base64.b64encode(proof.waitForResult()[1])
except TypeError:
# Happens when we failed to produce a proof
logger.error("Failed to produce a pow for announcing to " + peer)
announce_fail = True
else:
daemon.announceCache[peer] = data['random']
if not announce_fail:
logger.info('Announcing node to ' + url)
if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
logger.info('Successfully introduced node to ' + peer, terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
daemon.decrementThreadCount('announce_node')
return ret_data
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False):
"""Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
if i not in peerList and i not in kv.get('offlinePeers') \
and i not in gettransports.get() and len(str(i).strip()) > 0:
if not db_only:
peerList.append(i)
keydb.addkeys.add_address(i)
def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
'''
Add the bootstrap list to the peer list (no duplicates)
'''
for i in bootstrap_peers:
if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get(
) and len(str(i).strip()) > 0:
if not db_only: peerList.append(i)
keydb.addkeys.add_address(i)
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False):
"""Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
# Add bootstrap peers to peerList (does not save them)
# Don't add them if they're already added or in the offline list
if i not in peerList and i not in kv.get('offlinePeers') \
and i not in gettransports.get() and len(str(i).strip()) > 0:
if not db_only:
peerList.append(i)
keydb.addkeys.add_address(i)
def handle_announce(request):
'''
accept announcement posts, validating POW
clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
'''
resp = 'failure'
powHash = ''
randomData = ''
newNode = ''
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No node specified for upload')
pass
else:
try:
randomData = request.form['random']
randomData = base64.b64decode(randomData)
except KeyError:
logger.warn('No random data specified for upload')
else:
nodes = newNode + bytesconverter.str_to_bytes(
gettransports.get()[0])
nodes = crypto.hashers.blake2b_hash(nodes)
powHash = crypto.hashers.blake2b_hash(randomData + nodes)
try:
powHash = powHash.decode()
except AttributeError:
pass
if powHash.startswith('0' * onionrvalues.ANNOUNCE_POW):
newNode = bytesconverter.bytes_to_str(newNode)
announce_queue = deadsimplekv.DeadSimpleKV(
filepaths.announce_cache)
announce_queue_list = announce_queue.get('new_peers')
if announce_queue_list is None:
announce_queue_list = []
if stringvalidators.validate_transport(
newNode) and not newNode in announce_queue_list:
#clientAPI.onionrInst.communicatorInst.newPeers.append(newNode)
g.shared_state.get(
OnionrCommunicatorDaemon).newPeers.append(newNode)
announce_queue.put('new_peers',
announce_queue_list.append(newNode))
announce_queue.flush()
resp = 'Success'
else:
logger.warn(newNode.decode() + ' failed to meet POW: ' +
powHash)
resp = Response(resp)
if resp == 'failure':
return resp, 406
return resp
def validate_request():
'''Validate request has the correct hostname'''
# If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
transports = gettransports.get()
if public_api.config.get('general.security_level', default=1) > 0:
abort(403)
if request.host not in transports:
# Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
abort(403)
public_api.hitCount += 1 # raise hit count for valid requests
try:
if 'onionr' in request.headers['User-Agent'].lower():
g.is_onionr_client = True
else:
g.is_onionr_client = False
except KeyError:
g.is_onionr_client = False
def announce_node(daemon):
"""Announce our node to our peers."""
ret_data = False
kv: "DeadSimpleKV" = daemon.shared_state.get_by_string("DeadSimpleKV")
# Do not let announceCache get too large
if len(kv.get('announceCache')) >= 10000:
kv.get('announceCache').popitem()
if daemon.config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in kv.get('onlinePeers'):
if i not in kv.get('announceCache'):
peer = i
break
else:
try:
peer = onlinepeers.pick_online_peer(daemon)
except onionrexceptions.OnlinePeerNeeded:
peer = ""
try:
ourID = gettransports.get()[0]
if not peer:
raise onionrexceptions.OnlinePeerNeeded
except (IndexError, onionrexceptions.OnlinePeerNeeded):
pass
else:
url = 'http://' + peer + '/announce'
data = {'node': ourID}
logger.info('Announcing node to ' + url)
if basicrequests.do_post_request(
url,
data,
port=daemon.shared_state.get(NetController).socksPort)\
== 'Success':
logger.info('Successfully introduced node to ' + peer,
terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
daemon.decrementThreadCount('announce_node')
return ret_data
def lookup_new_peer_transports_with_communicator(shared_state):
logger.info('Looking up new addresses...')
tryAmount = 1
newPeers = []
transports = gettransports.get()
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Download new peer address list from random online peers
if len(newPeers) > 10000:
# Don't get new peers if we have too many queued up
break
try:
peer = onlinepeers.pick_online_peer(kv)
newAdders = peeraction.peer_action(shared_state,
peer,
action='pex')
except onionrexceptions.OnlinePeerNeeded:
continue
try:
newPeers = newAdders.split(',')
except AttributeError:
pass
else:
# Validate new peers are good format and not already in queue
invalid = []
for x in newPeers:
x = x.strip()
if not stringvalidators.validate_transport(x) \
or x in kv.get('newPeers') or x in transports:
# avoid adding if its our address
invalid.append(x)
for x in invalid:
try:
newPeers.remove(x)
except ValueError:
pass
kv.get('newPeers').extend(newPeers)
def _get_tor_adder(pub_api):
transports = []
while len(transports) == 0:
transports = gettransports.get()
time.sleep(0.3)
pub_api.torAdder = transports[0]
def connect_new_peer_to_communicator(shared_state,
peer='',
useBootstrap=False):
retData = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
tried = kv.get('offlinePeers')
transports = gettransports.get()
if peer != '':
if stringvalidators.validate_transport(peer):
peerList = [peer]
else:
raise onionrexceptions.InvalidAddress(
'Will not attempt connection test to invalid address')
else:
peerList = keydb.listkeys.list_adders()
mainPeerList = keydb.listkeys.list_adders()
peerList = onionrpeers.get_score_sorted_peer_list()
"""
If we don't have enough peers connected or random chance,
select new peers to try
"""
if len(peerList) < 8 or secrets.randbelow(4) == 3:
tryingNew = []
for x in kv.get('newPeers'):
if x not in peerList:
peerList.append(x)
tryingNew.append(x)
for i in tryingNew:
kv.get('newPeers').remove(i)
if len(peerList) == 0 or useBootstrap:
# Avoid duplicating bootstrap addresses in peerList
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList)
for address in peerList:
address = address.strip()
# Don't connect to our own address
if address in transports:
continue
"""Don't connect to invalid address or
if its already been tried/connected, or if its cooled down
"""
if len(address) == 0 or address in tried \
or address in kv.get('onlinePeers') \
or address in kv.get('cooldownPeer'):
continue
if kv.get('shutdown'):
return
# Ping a peer,
ret = peeraction.peer_action(shared_state, address, 'ping')
if ret == 'pong!':
time.sleep(0.1)
if address not in mainPeerList:
# Add a peer to our list if it isn't already since it connected
networkmerger.mergeAdders(address)
if address not in kv.get('onlinePeers'):
logger.info('Connected to ' + address, terminal=True)
kv.get('onlinePeers').append(address)
kv.get('connectTimes')[address] = epoch.get_epoch()
retData = address
# add peer to profile list if they're not in it
for profile in kv.get('peerProfiles'):
if profile.address == address:
break
else:
kv.get('peerProfiles').append(
onionrpeers.PeerProfiles(address))
break
else:
# Mark a peer as tried if they failed to respond to ping
tried.append(address)
logger.debug('Failed to connect to %s: %s ' % (address, ret))
return retData
