def mergeAdders(newAdderList): ''' Merge peer adders list to our database ''' blacklist = onionrblacklist.OnionrBlackList() retVal = False if newAdderList != False: for adder in newAdderList.split(','): adder = adder.strip() if not adder in keydb.listkeys.list_adders( randomOrder=False) and not adder in gettransports.get( ) and not blacklist.inBlacklist(adder): if keydb.addkeys.add_address(adder): # Check if we have the maximum amount of allowed stored peers if config.get('peers.max_stored_peers') > len( keydb.listkeys.list_adders()): logger.info('Added %s to db.' % adder, timestamp=True) retVal = True else: logger.warn( 'Reached the maximum amount of peers in the net database as allowed by your config.' ) else: pass #logger.debug('%s is either our address or already in our DB' % adder) return retVal
def add_address(address): """Add an address to the address database (only tor currently)""" if type(address) is None or len(address) == 0: return False if stringvalidators.validate_transport(address): if address in gettransports.get(): return False conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() # check if address is in database # this is safe to do because the address is validated above, but we strip some chars here too just in case address = address.replace('\'', '').replace(';', '').replace( '"', '').replace('\\', '') for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address, )): try: if i[0] == address: conn.close() return False except ValueError: pass except IndexError: pass t = (address, 1) c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t) conn.commit() conn.close() return True else: return False
def validate_request(): """Validate request has the correct hostname""" # If high security level, deny requests to public # (HS should be disabled anyway for Tor, but might not be for I2P) g.is_onionr_client = False transports = gettransports.get() if public_api.config.get('general.security_level', default=1) > 0: abort(403) if request.host not in transports: # Abort conn if wrong HTTP hostname, to prevent DNS rebinding abort(403) public_api.hitCount += 1 # raise hit count for valid requests try: if 'onionr' in request.headers['User-Agent'].lower(): g.is_onionr_client = True else: g.is_onionr_client = False except KeyError: g.is_onionr_client = False # Add shared objects try: g.too_many = public_api._too_many except KeyError: g.too_many = None
def lookup_new_peer_transports_with_communicator(comm_inst): logger.info('Looking up new addresses...') tryAmount = 1 newPeers = [] transports = gettransports.get() for i in range(tryAmount): # Download new peer address list from random online peers if len(newPeers) > 10000: # Don't get new peers if we have too many queued up break peer = onlinepeers.pick_online_peer(comm_inst) newAdders = peeraction.peer_action(comm_inst, peer, action='pex') try: newPeers = newAdders.split(',') except AttributeError: pass else: # Validate new peers are good format and not already in queue invalid = [] for x in newPeers: x = x.strip() if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports: # avoid adding if its our address invalid.append(x) for x in invalid: try: newPeers.remove(x) except ValueError: pass comm_inst.newPeers.extend(newPeers) comm_inst.decrementThreadCount('lookup_new_peer_transports_with_communicator')
def announce_node(daemon): '''Announce our node to our peers''' ret_data = False announce_fail = False # Do not let announceCache get too large if len(daemon.announceCache) >= 10000: daemon.announceCache.popitem() if daemon.config.get('general.security_level', 0) == 0: # Announce to random online peers for i in daemon.onlinePeers: if not i in daemon.announceCache and not i in daemon.announceProgress: peer = i break else: peer = onlinepeers.pick_online_peer(daemon) for x in range(1): try: ourID = gettransports.get()[0] except IndexError: break url = 'http://' + peer + '/announce' data = {'node': ourID} combinedNodes = ourID + peer if ourID != 1: existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue')) # Reset existingRand if it no longer meets the minimum POW if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW): existingRand = '' if peer in daemon.announceCache: data['random'] = daemon.announceCache[peer] elif len(existingRand) > 0: data['random'] = existingRand else: daemon.announceProgress[peer] = True proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW) del daemon.announceProgress[peer] try: data['random'] = base64.b64encode(proof.waitForResult()[1]) except TypeError: # Happens when we failed to produce a proof logger.error("Failed to produce a pow for announcing to " + peer) announce_fail = True else: daemon.announceCache[peer] = data['random'] if not announce_fail: logger.info('Announcing node to ' + url) if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success': logger.info('Successfully introduced node to ' + peer, terminal=True) ret_data = True keydb.transportinfo.set_address_info(peer, 'introduced', 1) keydb.transportinfo.set_address_info(peer, 'powValue', data['random']) daemon.decrementThreadCount('announce_node') return ret_data
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False): """Add the bootstrap list to the peer list (no duplicates).""" for i in bootstrap_peers: if i not in peerList and i not in kv.get('offlinePeers') \ and i not in gettransports.get() and len(str(i).strip()) > 0: if not db_only: peerList.append(i) keydb.addkeys.add_address(i)
def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False): ''' Add the bootstrap list to the peer list (no duplicates) ''' for i in bootstrap_peers: if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get( ) and len(str(i).strip()) > 0: if not db_only: peerList.append(i) keydb.addkeys.add_address(i)
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False): """Add the bootstrap list to the peer list (no duplicates).""" for i in bootstrap_peers: # Add bootstrap peers to peerList (does not save them) # Don't add them if they're already added or in the offline list if i not in peerList and i not in kv.get('offlinePeers') \ and i not in gettransports.get() and len(str(i).strip()) > 0: if not db_only: peerList.append(i) keydb.addkeys.add_address(i)
def handle_announce(request): ''' accept announcement posts, validating POW clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request ''' resp = 'failure' powHash = '' randomData = '' newNode = '' try: newNode = request.form['node'].encode() except KeyError: logger.warn('No node specified for upload') pass else: try: randomData = request.form['random'] randomData = base64.b64decode(randomData) except KeyError: logger.warn('No random data specified for upload') else: nodes = newNode + bytesconverter.str_to_bytes( gettransports.get()[0]) nodes = crypto.hashers.blake2b_hash(nodes) powHash = crypto.hashers.blake2b_hash(randomData + nodes) try: powHash = powHash.decode() except AttributeError: pass if powHash.startswith('0' * onionrvalues.ANNOUNCE_POW): newNode = bytesconverter.bytes_to_str(newNode) announce_queue = deadsimplekv.DeadSimpleKV( filepaths.announce_cache) announce_queue_list = announce_queue.get('new_peers') if announce_queue_list is None: announce_queue_list = [] if stringvalidators.validate_transport( newNode) and not newNode in announce_queue_list: #clientAPI.onionrInst.communicatorInst.newPeers.append(newNode) g.shared_state.get( OnionrCommunicatorDaemon).newPeers.append(newNode) announce_queue.put('new_peers', announce_queue_list.append(newNode)) announce_queue.flush() resp = 'Success' else: logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) resp = Response(resp) if resp == 'failure': return resp, 406 return resp
def validate_request(): '''Validate request has the correct hostname''' # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P) transports = gettransports.get() if public_api.config.get('general.security_level', default=1) > 0: abort(403) if request.host not in transports: # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks abort(403) public_api.hitCount += 1 # raise hit count for valid requests try: if 'onionr' in request.headers['User-Agent'].lower(): g.is_onionr_client = True else: g.is_onionr_client = False except KeyError: g.is_onionr_client = False
def announce_node(daemon): """Announce our node to our peers.""" ret_data = False kv: "DeadSimpleKV" = daemon.shared_state.get_by_string("DeadSimpleKV") # Do not let announceCache get too large if len(kv.get('announceCache')) >= 10000: kv.get('announceCache').popitem() if daemon.config.get('general.security_level', 0) == 0: # Announce to random online peers for i in kv.get('onlinePeers'): if i not in kv.get('announceCache'): peer = i break else: try: peer = onlinepeers.pick_online_peer(daemon) except onionrexceptions.OnlinePeerNeeded: peer = "" try: ourID = gettransports.get()[0] if not peer: raise onionrexceptions.OnlinePeerNeeded except (IndexError, onionrexceptions.OnlinePeerNeeded): pass else: url = 'http://' + peer + '/announce' data = {'node': ourID} logger.info('Announcing node to ' + url) if basicrequests.do_post_request( url, data, port=daemon.shared_state.get(NetController).socksPort)\ == 'Success': logger.info('Successfully introduced node to ' + peer, terminal=True) ret_data = True keydb.transportinfo.set_address_info(peer, 'introduced', 1) daemon.decrementThreadCount('announce_node') return ret_data
def lookup_new_peer_transports_with_communicator(shared_state): logger.info('Looking up new addresses...') tryAmount = 1 newPeers = [] transports = gettransports.get() kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") for i in range(tryAmount): # Download new peer address list from random online peers if len(newPeers) > 10000: # Don't get new peers if we have too many queued up break try: peer = onlinepeers.pick_online_peer(kv) newAdders = peeraction.peer_action(shared_state, peer, action='pex') except onionrexceptions.OnlinePeerNeeded: continue try: newPeers = newAdders.split(',') except AttributeError: pass else: # Validate new peers are good format and not already in queue invalid = [] for x in newPeers: x = x.strip() if not stringvalidators.validate_transport(x) \ or x in kv.get('newPeers') or x in transports: # avoid adding if its our address invalid.append(x) for x in invalid: try: newPeers.remove(x) except ValueError: pass kv.get('newPeers').extend(newPeers)
def _get_tor_adder(pub_api): transports = [] while len(transports) == 0: transports = gettransports.get() time.sleep(0.3) pub_api.torAdder = transports[0]
def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False): retData = False kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") tried = kv.get('offlinePeers') transports = gettransports.get() if peer != '': if stringvalidators.validate_transport(peer): peerList = [peer] else: raise onionrexceptions.InvalidAddress( 'Will not attempt connection test to invalid address') else: peerList = keydb.listkeys.list_adders() mainPeerList = keydb.listkeys.list_adders() peerList = onionrpeers.get_score_sorted_peer_list() """ If we don't have enough peers connected or random chance, select new peers to try """ if len(peerList) < 8 or secrets.randbelow(4) == 3: tryingNew = [] for x in kv.get('newPeers'): if x not in peerList: peerList.append(x) tryingNew.append(x) for i in tryingNew: kv.get('newPeers').remove(i) if len(peerList) == 0 or useBootstrap: # Avoid duplicating bootstrap addresses in peerList if config.get('general.use_bootstrap_list', True): bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList) for address in peerList: address = address.strip() # Don't connect to our own address if address in transports: continue """Don't connect to invalid address or if its already been tried/connected, or if its cooled down """ if len(address) == 0 or address in tried \ or address in kv.get('onlinePeers') \ or address in kv.get('cooldownPeer'): continue if kv.get('shutdown'): return # Ping a peer, ret = peeraction.peer_action(shared_state, address, 'ping') if ret == 'pong!': time.sleep(0.1) if address not in mainPeerList: # Add a peer to our list if it isn't already since it connected networkmerger.mergeAdders(address) if address not in kv.get('onlinePeers'): logger.info('Connected to ' + address, terminal=True) kv.get('onlinePeers').append(address) kv.get('connectTimes')[address] = epoch.get_epoch() retData = address # add peer to profile list if they're not in it for profile in kv.get('peerProfiles'): if profile.address == address: break else: kv.get('peerProfiles').append( onionrpeers.PeerProfiles(address)) break else: # Mark a peer as tried if they failed to respond to ping tried.append(address) logger.debug('Failed to connect to %s: %s ' % (address, ret)) return retData