def ban_block(): """Deletes a block, permanently blacklisting it""" blacklist = onionrblacklist.OnionrBlackList() try: ban = sys.argv[2] except IndexError: # Get the hash if its not provided as a CLI argument ban = logger.readline('Enter a block hash:').strip() # Make sure the hash has no truncated zeroes ban = reconstructhash.reconstruct_hash(ban) if stringvalidators.validate_hash(ban): if not blacklist.inBlacklist(ban): try: blacklist.addToDB(ban) removeblock.remove_block(ban) deleteBlock(ban) except Exception as error: logger.error('Could not blacklist block', error=error, terminal=True) else: logger.info('Block blacklisted', terminal=True) else: logger.warn('That block is already blacklisted', terminal=True) else: logger.error('Invalid block hash', terminal=True)
def mergeAdders(newAdderList): ''' Merge peer adders list to our database ''' blacklist = onionrblacklist.OnionrBlackList() retVal = False if newAdderList != False: for adder in newAdderList.split(','): adder = adder.strip() if not adder in keydb.listkeys.list_adders( randomOrder=False) and not adder in gettransports.get( ) and not blacklist.inBlacklist(adder): if keydb.addkeys.add_address(adder): # Check if we have the maximum amount of allowed stored peers if config.get('peers.max_stored_peers') > len( keydb.listkeys.list_adders()): logger.info('Added %s to db.' % adder, timestamp=True) retVal = True else: logger.warn( 'Reached the maximum amount of peers in the net database as allowed by your config.' ) else: pass #logger.debug('%s is either our address or already in our DB' % adder) return retVal
def peer_cleanup(): '''Removes peers who have been offline too long or score too low''' logger.info('Cleaning peers...') blacklist = onionrblacklist.OnionrBlackList() adders = scoresortedpeerlist.get_score_sorted_peer_list() adders.reverse() if len(adders) > 1: min_score = int(config.get('peers.minimum_score', -100)) max_peers = int(config.get('peers.max_stored', 5000)) for address in adders: # Remove peers that go below the negative score if peerprofiles.PeerProfiles(address).score < min_score: keydb.removekeys.remove_address(address) try: lastConnect = int( keydb.transportinfo.get_address_info( address, 'lastConnect')) expireTime = 86400 - int(epoch.get_epoch()) - lastConnect blacklist.addToDB(address, dataType=1, expire=expireTime) except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue pass except ValueError: pass logger.warn('Removed address ' + address + '.') # Unban probably not malicious peers TODO improve blacklist.deleteExpired(dataType=1)
def __purge_block(shared_state, block_hash, add_to_blacklist = True): blacklist = None removeblock.remove_block(block_hash) onionrstorage.deleteBlock(block_hash) __remove_from_upload(shared_state, block_hash) if add_to_blacklist: blacklist = onionrblacklist.OnionrBlackList() blacklist.addToDB(block_hash)
def __init__(self, address): if not stringvalidators.validate_transport(address): raise onionrexceptions.InvalidAddress self.address = address # node address self.score = None self.friendSigCount = 0 self.success = 0 self.failure = 0 self.connectTime = None self.loadScore() self.getConnectTime() self.last_updated = {'connect_time': UPDATE_DELAY, 'score': UPDATE_DELAY} # Last time a given value was updated if not address in keydb.listkeys.list_adders() and not onionrblacklist.OnionrBlackList().inBlacklist(address): keydb.addkeys.add_address(address)
def clean_old_blocks(shared_state): """Delete expired blocks + old blocks if disk allocation is near full""" blacklist = onionrblacklist.OnionrBlackList() # Delete expired blocks for bHash in blockmetadb.expiredblocks.get_expired_blocks(): __purge_block(shared_state, bHash, add_to_blacklist=True) logger.info('Deleted expired block: %s' % (bHash,)) while storage_counter.is_full(): try: oldest = blockmetadb.get_block_list()[0] except IndexError: break else: __purge_block(shared_state, bHash, add_to_blacklist=True) logger.info('Deleted block because of full storage: %s' % (oldest,))
def should_download(comm_inst, block_hash) -> bool: """Return bool for if a (assumed to exist) block should be downloaded.""" blacklist = onionrblacklist.OnionrBlackList() should = True if block_hash in blockmetadb.get_block_list(): # Don't download block we have should = False else: if blacklist.inBlacklist(block_hash): # Don't download blacklisted block should = False if should is False: # Remove block from communicator queue if it shouldn't be downloaded try: del comm_inst.blockQueue[block_hash] except KeyError: pass return should
def should_download(shared_state, block_hash) -> bool: """Return bool for if a (assumed to exist) block should be downloaded.""" blacklist = onionrblacklist.OnionrBlackList() should = True kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") if block_hash in blockmetadb.get_block_list(): # Don't download block we have should = False else: if blacklist.inBlacklist(block_hash): # Don't download blacklisted block should = False if should is False: # Remove block from communicator queue if it shouldn't be downloaded try: del kv.get('blockQueue')[block_hash] except KeyError: pass return should
def clean_old_blocks(comm_inst): '''Delete old blocks if our disk allocation is full/near full, and also expired blocks''' blacklist = onionrblacklist.OnionrBlackList() # Delete expired blocks for bHash in blockmetadb.expiredblocks.get_expired_blocks(): blacklist.addToDB(bHash) removeblock.remove_block(bHash) onionrstorage.deleteBlock(bHash) __remove_from_upload(comm_inst, bHash) logger.info('Deleted block: %s' % (bHash, )) while comm_inst.storage_counter.is_full(): oldest = blockmetadb.get_block_list()[0] blacklist.addToDB(oldest) removeblock.remove_block(oldest) onionrstorage.deleteBlock(oldest) __remove_from_upload.remove(comm_inst, oldest) logger.info('Deleted block: %s' % (oldest, )) comm_inst.decrementThreadCount('clean_old_blocks')
def clean_old_blocks(comm_inst): """Delete expired blocks + old blocks if disk allocation is near full""" blacklist = onionrblacklist.OnionrBlackList() # Delete expired blocks for bHash in blockmetadb.expiredblocks.get_expired_blocks(): blacklist.addToDB(bHash) removeblock.remove_block(bHash) onionrstorage.deleteBlock(bHash) __remove_from_upload(comm_inst, bHash) logger.info('Deleted block: %s' % (bHash, )) while storage_counter.is_full(): try: oldest = blockmetadb.get_block_list()[0] except IndexError: break else: blacklist.addToDB(oldest) removeblock.remove_block(oldest) onionrstorage.deleteBlock(oldest) __remove_from_upload(comm_inst, oldest) logger.info('Deleted block: %s' % (oldest, )) comm_inst.decrementThreadCount('clean_old_blocks')
def download_blocks_from_communicator(shared_state: "TooMany"): """Use communicator instance to download blocks in the comms's queue""" blacklist = onionrblacklist.OnionrBlackList() kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter count: int = 0 metadata_validation_result: bool = False # Iterate the block queue in the communicator for blockHash in list(kv.get('blockQueue')): count += 1 try: blockPeers = list(kv.get('blockQueue')[blockHash]) except KeyError: blockPeers = [] removeFromQueue = True if not shoulddownload.should_download(shared_state, blockHash): continue if kv.get('shutdown') or not kv.get('isOnline') or \ storage_counter.is_full(): # Exit loop if shutting down or offline, or disk allocation reached break # Do not download blocks being downloaded if blockHash in kv.get('currentDownloading'): continue if len(kv.get('onlinePeers')) == 0: break # So we can avoid concurrent downloading in other threads of same block kv.get('currentDownloading').append(blockHash) if len(blockPeers) == 0: try: peerUsed = onlinepeers.pick_online_peer(kv) except onionrexceptions.OnlinePeerNeeded: continue else: SystemRandom().shuffle(blockPeers) peerUsed = blockPeers.pop(0) if not kv.get('shutdown') and peerUsed.strip() != '': logger.info(f"Attempting to download %s from {peerUsed}..." % (blockHash[:12], )) content = peeraction.peer_action( shared_state, peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer if content is not False and len(content) > 0: try: content = content.encode() except AttributeError: pass realHash = onionrcrypto.hashers.sha3_hash(content) try: realHash = realHash.decode( ) # bytes on some versions for some reason except AttributeError: pass if realHash == blockHash: #content = content.decode() # decode here because sha3Hash needs bytes above metas = blockmetadata.get_block_metadata_from_data( content ) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] try: metadata_validation_result = \ validatemetadata.validate_metadata(metadata, metas[2]) except onionrexceptions.PlaintextNotSupported: logger.debug( f"Not saving {blockHash} due to plaintext not enabled") removeFromQueue = True except onionrexceptions.DataExists: metadata_validation_result = False if metadata_validation_result: # check if metadata is valid, and verify nonce if onionrcrypto.cryptoutils.verify_POW( content): # check if POW is enough/correct logger.info('Attempting to save block %s...' % blockHash[:12]) try: onionrstorage.set_data(content) except onionrexceptions.DataExists: logger.warn('Data is already set for %s ' % (blockHash, )) except onionrexceptions.DiskAllocationReached: logger.error( 'Reached disk allocation allowance, cannot save block %s.' % (blockHash, )) removeFromQueue = False else: blockmetadb.add_to_block_DB( blockHash, dataSaved=True) # add block to meta db blockmetadata.process_block_metadata( blockHash ) # caches block metadata values to block database spawn(local_command, f'/daemon-event/upload_event', post=True, is_json=True, post_data={'block': blockHash}) else: logger.warn('POW failed for block %s.' % (blockHash, )) else: if blacklist.inBlacklist(realHash): logger.warn('Block %s is blacklisted.' % (realHash, )) else: logger.warn('Metadata for block %s is invalid.' % (blockHash, )) blacklist.addToDB(blockHash) else: # if block didn't meet expected hash tempHash = onionrcrypto.hashers.sha3_hash( content) # lazy hack, TODO use var try: tempHash = tempHash.decode() except AttributeError: pass # Punish peer for sharing invalid block (not always malicious, but is bad regardless) onionrpeers.PeerProfiles(peerUsed).addScore(-50) if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253': # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error. logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) else: removeFromQueue = False # Don't remove from queue if 404 if removeFromQueue: try: del kv.get( 'blockQueue' )[blockHash] # remove from block queue both if success or false if count == LOG_SKIP_COUNT: logger.info('%s blocks remaining in queue' % [len(kv.get('blockQueue'))], terminal=True) count = 0 except KeyError: pass kv.get('currentDownloading').remove(blockHash)
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. ''' import logger, onionrproofs from onionrutils import stringvalidators, epoch from communicator import peeraction, onlinepeers from coredb import blockmetadb from utils import reconstructhash from onionrblocks import onionrblacklist blacklist = onionrblacklist.OnionrBlackList() def lookup_blocks_from_communicator(comm_inst): logger.info('Looking up new blocks') tryAmount = 2 newBlocks = '' existingBlocks = blockmetadb.get_block_list() # List of existing saved blocks triedPeers = [] # list of peers we've tried this time around maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion lastLookupTime = 0 # Last time we looked up a particular peer's list new_block_count = 0 for i in range(tryAmount): listLookupCommand = 'getblocklist' # This is defined here to reset it each time if len(comm_inst.blockQueue) >= maxBacklog: break if not comm_inst.isOnline: break
def show_stats(): """Print/log statistic info about our Onionr install.""" try: # define stats messages here totalBlocks = len(blockmetadb.get_block_list()) home = identifyhome.identify_home() totalBanned = len(onionrblacklist.OnionrBlackList().getList()) messages = { # info about local client 'Onionr Daemon Status': ((logger.colors.fg.green + 'Online') if check_communicator( timeout=9) else logger.colors.fg.red + 'Offline'), # file and folder size stats 'div1': True, # this creates a solid line across the screen, a div 'Total Block Size': sizeutils.human_size(sizeutils.size(home + 'blocks/')), 'Total Plugin Size': sizeutils.human_size(sizeutils.size(home + 'plugins/')), 'Log File Size': sizeutils.human_size(sizeutils.size(home + 'output.log')), # count stats 'div2': True, 'Known Peers (nodes)': str(max(len(keydb.listkeys.list_adders()) - 1, 0)), 'Enabled Plugins': str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(home + 'plugins/'))), 'Stored Blocks': str(totalBlocks), 'Deleted Blocks': str(totalBanned) } # color configuration colors = { 'title': logger.colors.bold, 'key': logger.colors.fg.lightgreen, 'val': logger.colors.fg.green, 'border': logger.colors.fg.lightblue, 'reset': logger.colors.reset } # pre-processing maxlength = 0 width = getconsolewidth.get_console_width() for key, val in messages.items(): if not (type(val) is bool and val is True): maxlength = max(len(key), maxlength) prewidth = maxlength + len(' | ') groupsize = width - prewidth - len('[+] ') # generate stats table logger.info(colors['title'] + 'Onionr v%s Statistics' % onionrvalues.ONIONR_VERSION + colors['reset'], terminal=True) logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True) for key, val in messages.items(): if not (type(val) is bool and val is True): val = [ str(val)[i:i + groupsize] for i in range(0, len(str(val)), groupsize) ] logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(val.pop(0)) + colors['reset'], terminal=True) for value in val: logger.info(' ' * maxlength + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(value) + colors['reset'], terminal=True) else: logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True) logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True) except Exception as e: # pylint: disable=W0703 logger.error('Failed to generate statistics table. ' + str(e), error=e, timestamp=False, terminal=True)
def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"): '''Use communicator instance to download blocks in the comms's queue''' blacklist = onionrblacklist.OnionrBlackList() storage_counter = storagecounter.StorageCounter() LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter count: int = 0 metadata_validation_result: bool = False # Iterate the block queue in the communicator for blockHash in list(comm_inst.blockQueue): count += 1 if len(comm_inst.onlinePeers) == 0: break triedQueuePeers = [] # List of peers we've tried for a block try: blockPeers = list(comm_inst.blockQueue[blockHash]) except KeyError: blockPeers = [] removeFromQueue = True if not shoulddownload.should_download(comm_inst, blockHash): continue if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full( ): # Exit loop if shutting down or offline, or disk allocation reached break # Do not download blocks being downloaded if blockHash in comm_inst.currentDownloading: continue comm_inst.currentDownloading.append( blockHash ) # So we can avoid concurrent downloading in other threads of same block if len(blockPeers) == 0: peerUsed = onlinepeers.pick_online_peer(comm_inst) else: blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers) peerUsed = blockPeers.pop(0) if not comm_inst.shutdown and peerUsed.strip() != '': logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) content = peeraction.peer_action( comm_inst, peerUsed, 'getdata/' + blockHash, max_resp_size=3000000 ) # block content from random peer (includes metadata) if content != False and len(content) > 0: try: content = content.encode() except AttributeError: pass realHash = onionrcrypto.hashers.sha3_hash(content) try: realHash = realHash.decode( ) # bytes on some versions for some reason except AttributeError: pass if realHash == blockHash: #content = content.decode() # decode here because sha3Hash needs bytes above metas = blockmetadata.get_block_metadata_from_data( content ) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] try: metadata_validation_result = validatemetadata.validate_metadata( metadata, metas[2]) except onionrexceptions.DataExists: metadata_validation_result = False if metadata_validation_result: # check if metadata is valid, and verify nonce if onionrcrypto.cryptoutils.verify_POW( content): # check if POW is enough/correct logger.info('Attempting to save block %s...' % blockHash[:12]) try: onionrstorage.set_data(content) except onionrexceptions.DataExists: logger.warn('Data is already set for %s ' % (blockHash, )) except onionrexceptions.DiskAllocationReached: logger.error( 'Reached disk allocation allowance, cannot save block %s.' % (blockHash, )) removeFromQueue = False else: blockmetadb.add_to_block_DB( blockHash, dataSaved=True) # add block to meta db blockmetadata.process_block_metadata( blockHash ) # caches block metadata values to block database else: logger.warn('POW failed for block %s.' % (blockHash, )) else: if blacklist.inBlacklist(realHash): logger.warn('Block %s is blacklisted.' % (realHash, )) else: logger.warn('Metadata for block %s is invalid.' % (blockHash, )) blacklist.addToDB(blockHash) else: # if block didn't meet expected hash tempHash = onionrcrypto.hashers.sha3_hash( content) # lazy hack, TODO use var try: tempHash = tempHash.decode() except AttributeError: pass # Punish peer for sharing invalid block (not always malicious, but is bad regardless) onionrpeers.PeerProfiles(peerUsed).addScore(-50) if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253': # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error. logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) else: removeFromQueue = False # Don't remove from queue if 404 if removeFromQueue: try: del comm_inst.blockQueue[ blockHash] # remove from block queue both if success or false if count == LOG_SKIP_COUNT: logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True) count = 0 except KeyError: pass comm_inst.currentDownloading.remove(blockHash) comm_inst.decrementThreadCount('getBlocks')