def get_block_data(public_API, b_hash): """return block data by hash unless we are hiding it""" resp = '' b_hash = reconstructhash.reconstruct_hash(b_hash) if stringvalidators.validate_hash(b_hash): if not config.get('general.hide_created_blocks', True) \ or b_hash not in public_API.hideBlocks: if b_hash in public_API._too_many.get(BlockList).get(): block = apiutils.GetBlockData().get_block_data(b_hash, raw=True, decrypt=False) try: # Encode in case data is binary block = block.encode('utf-8') except AttributeError: # 404 if no block data if not block: abort(404) if not len(block): abort(404) resp = block if len(resp) == 0: abort(404) resp = "" # Has to be octet stream, otherwise binary data fails hash check return Response(resp, mimetype='application/octet-stream')
def clean_session(self, specific_session: Union[str, UploadSession] = None): comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string( "OnionrCommunicatorDaemon") sessions_to_delete = [] if comm_inst.getUptime() < 120: return onlinePeerCount = len(comm_inst.onlinePeers) # If we have no online peers right now, if onlinePeerCount == 0: return for session in self.sessions: # if over 50% of peers that were online for a session have become unavailable, don't kill sessions if session.total_success_count > onlinePeerCount: if onlinePeerCount / session.total_success_count >= 0.5: return # Clean sessions if they have uploaded to enough online peers if session.total_success_count <= 0: continue if (session.total_success_count / onlinePeerCount ) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: sessions_to_delete.append(session) for session in sessions_to_delete: self.sessions.remove(session) # TODO cleanup to one round of search # Remove the blocks from the sessions, upload list, and waitforshare list try: comm_inst.blocksToUpload.remove( reconstructhash.reconstruct_hash(session.block_hash)) except ValueError: pass try: comm_inst.blocksToUpload.remove(session.block_hash) except ValueError: pass localcommand.local_command('waitforshare/{session.block_hash}')
def ban_block(): """Deletes a block, permanently blacklisting it""" blacklist = onionrblacklist.OnionrBlackList() try: ban = sys.argv[2] except IndexError: # Get the hash if its not provided as a CLI argument ban = logger.readline('Enter a block hash:').strip() # Make sure the hash has no truncated zeroes ban = reconstructhash.reconstruct_hash(ban) if stringvalidators.validate_hash(ban): if not blacklist.inBlacklist(ban): try: blacklist.addToDB(ban) removeblock.remove_block(ban) deleteBlock(ban) except Exception as error: logger.error('Could not blacklist block', error=error, terminal=True) else: logger.info('Block blacklisted', terminal=True) else: logger.warn('That block is already blacklisted', terminal=True) else: logger.error('Invalid block hash', terminal=True)
def add_deleted(keyStore, b_hash): existing = keyStore.get('deleted_mail') bHash = reconstructhash.reconstruct_hash(b_hash) if existing is None: existing = [] else: if bHash in existing: return keyStore.put('deleted_mail', existing.append(b_hash))
def waitforshare(name): '''Used to prevent the **public** api from sharing blocks we just created''' if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric') name = reconstructhash.reconstruct_hash(name) if name in client_api.publicAPI.hideBlocks: client_api.publicAPI.hideBlocks.remove(name) return Response("removed") else: client_api.publicAPI.hideBlocks.append(name) return Response("added")
def __init__(self, block_hash: Union[str, bytes]): block_hash = bytesconverter.bytes_to_str(block_hash) block_hash = reconstructhash.reconstruct_hash(block_hash) if not stringvalidators.validate_hash(block_hash): raise ValueError self.start_time = epoch.get_epoch() self.block_hash = reconstructhash.deconstruct_hash(block_hash) self.total_fail_count: int = 0 self.total_success_count: int = 0 self.peer_fails = {} self.peer_exists = {}
def wait_for_share(name): """Prevent the **public** api from sharing blocks. Used for blocks we created usually """ if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric') name = reconstructhash.reconstruct_hash(name) if name in client_api.publicAPI.hideBlocks: return Response("will be removed") else: client_api.publicAPI.hideBlocks.append(name) return Response("added")
def clean_session(self, specific_session: Union[str, 'UploadSession'] = None): comm_inst: 'OnionrCommunicatorDaemon' # type: ignore comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore "OnionrCommunicatorDaemon") kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string( "DeadSimpleKV") sessions_to_delete = [] if kv.get('startTime') < 120: return onlinePeerCount = len(kv.get('onlinePeers')) # If we have no online peers right now, if onlinePeerCount == 0: return for sess in self.sessions: # if over 50% of peers that were online for a session have # become unavailable, don't kill sessions if sess.total_success_count > onlinePeerCount: if onlinePeerCount / sess.total_success_count >= 0.5: return # Clean sessions if they have uploaded to enough online peers if sess.total_success_count <= 0: continue if (sess.total_success_count / onlinePeerCount) >= \ onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: sessions_to_delete.append(sess) for sess in sessions_to_delete: try: self.sessions.remove(session) except ValueError: pass # TODO cleanup to one round of search # Remove the blocks from the sessions, upload list, # and waitforshare list try: kv.get('blocksToUpload').remove( reconstructhash.reconstruct_hash(sess.block_hash)) except ValueError: pass try: kv.get('blocksToUpload').remove(sess.block_hash) except ValueError: pass
def lookup_blocks_from_communicator(comm_inst): logger.info('Looking up new blocks') tryAmount = 2 newBlocks = '' existingBlocks = blockmetadb.get_block_list() # List of existing saved blocks triedPeers = [] # list of peers we've tried this time around maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion lastLookupTime = 0 # Last time we looked up a particular peer's list new_block_count = 0 for i in range(tryAmount): listLookupCommand = 'getblocklist' # This is defined here to reset it each time if len(comm_inst.blockQueue) >= maxBacklog: break if not comm_inst.isOnline: break # check if disk allocation is used if comm_inst.storage_counter.is_full(): logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used') break peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer # if we've already tried all the online peers this time around, stop if peer in triedPeers: if len(comm_inst.onlinePeers) == len(triedPeers): break else: continue triedPeers.append(peer) # Get the last time we looked up a peer's stamp to only fetch blocks since then. # Saved in memory only for privacy reasons try: lastLookupTime = comm_inst.dbTimestamps[peer] except KeyError: lastLookupTime = 0 else: listLookupCommand += '?date=%s' % (lastLookupTime,) try: newBlocks = peeraction.peer_action(comm_inst, peer, listLookupCommand) # get list of new block hashes except Exception as error: logger.warn('Could not get new blocks from %s.' % peer, error = error) newBlocks = False else: comm_inst.dbTimestamps[peer] = epoch.get_rounded_epoch(roundS=60) if newBlocks != False: # if request was a success for i in newBlocks.split('\n'): if stringvalidators.validate_hash(i): i = reconstructhash.reconstruct_hash(i) # if newline seperated string is valid hash if not i in existingBlocks: # if block does not exist on disk and is not already in block queue if i not in comm_inst.blockQueue: if onionrproofs.hashMeetsDifficulty(i) and not blacklist.inBlacklist(i): if len(comm_inst.blockQueue) <= 1000000: comm_inst.blockQueue[i] = [peer] # add blocks to download queue new_block_count += 1 else: if peer not in comm_inst.blockQueue[i]: if len(comm_inst.blockQueue[i]) < 10: comm_inst.blockQueue[i].append(peer) if new_block_count > 0: block_string = "" if new_block_count > 1: block_string = "s" logger.info('Discovered %s new block%s' % (new_block_count, block_string), terminal=True) comm_inst.download_blocks_timer.count = int(comm_inst.download_blocks_timer.frequency * 0.99) comm_inst.decrementThreadCount('lookup_blocks_from_communicator') return
def lookup_blocks_from_communicator(comm_inst): logger.info('Looking up new blocks') tryAmount = 2 newBlocks = '' # List of existing saved blocks existingBlocks = get_block_list() triedPeers = [] # list of peers we've tried this time around # Max amount of *new* block hashes to have in queue maxBacklog = 1560 lastLookupTime = 0 # Last time we looked up a particular peer's list new_block_count = 0 for i in range(tryAmount): # Defined here to reset it each time, time offset is added later listLookupCommand = 'getblocklist' if len(comm_inst.blockQueue) >= maxBacklog: break if not comm_inst.isOnline: break # check if disk allocation is used if comm_inst.storage_counter.is_full(): logger.debug( 'Not looking up new blocks due to maximum amount of disk used') break try: # select random online peer peer = onlinepeers.pick_online_peer(comm_inst) except onionrexceptions.OnlinePeerNeeded: time.sleep(1) continue # if we've already tried all the online peers this time around, stop if peer in triedPeers: if len(comm_inst.onlinePeers) == len(triedPeers): break else: continue triedPeers.append(peer) # Get the last time we looked up a peer's stamp, # to only fetch blocks since then. # Saved in memory only for privacy reasons try: lastLookupTime = comm_inst.dbTimestamps[peer] except KeyError: lastLookupTime = epoch.get_epoch() - \ config.get("general.max_block_age", onionrvalues.DEFAULT_EXPIRE) listLookupCommand += '?date=%s' % (lastLookupTime,) try: newBlocks = peeraction.peer_action( comm_inst, peer, listLookupCommand) # get list of new block hashes except Exception as error: logger.warn( f'Could not get new blocks from {peer}.', error=error) newBlocks = False if newBlocks != False: # noqa # if request was a success for i in newBlocks.split('\n'): if stringvalidators.validate_hash(i): i = reconstructhash.reconstruct_hash(i) # if newline seperated string is valid hash # if block does not exist on disk + is not already in queue if i not in existingBlocks: if i not in comm_inst.blockQueue: if onionrproofs.hashMeetsDifficulty(i) and \ not blacklist.inBlacklist(i): if len(comm_inst.blockQueue) <= 1000000: # add blocks to download queue comm_inst.blockQueue[i] = [peer] new_block_count += 1 comm_inst.dbTimestamps[peer] = \ epoch.get_rounded_epoch(roundS=60) else: if peer not in comm_inst.blockQueue[i]: if len(comm_inst.blockQueue[i]) < 10: comm_inst.blockQueue[i].append(peer) if new_block_count > 0: block_string = "" if new_block_count > 1: block_string = "s" logger.info( f'Discovered {new_block_count} new block{block_string}', terminal=True) comm_inst.download_blocks_timer.count = \ int(comm_inst.download_blocks_timer.frequency * 0.99) comm_inst.decrementThreadCount('lookup_blocks_from_communicator')
def test_reconstruct(self): h = b"4d20d791cbc293999b97cc627aa011692d317dede3d0fbd390c763210b0d" self.assertEqual(reconstructhash.reconstruct_hash(h), b"0000" + h) h = b"4d20d791cbc293999b97cc627aa011692d317dede3d0fbd390c763210b0d" self.assertEqual(reconstructhash.reconstruct_hash(h, 62), b"00" + h)