def site_file(name: str, file: str)->Response: """Accept a site 'name', if pubkey then show multi-page site, if hash show single page site""" resp: str = 'Not Found' mime_type = mimetypes.MimeTypes().guess_type(file)[0] # If necessary convert the name to base32 from mnemonic if mnemonickeys.DELIMITER in name: name = mnemonickeys.get_base32(name) # Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary) if stringvalidators.validate_pub_key(name): name = unpaddedbase32.repad(name) resp = sitefiles.get_file(name, file) elif stringvalidators.validate_hash(name): try: resp = onionrblockapi.Block(name).bcontent except onionrexceptions.NoDataAvailable: abort(404) except TypeError: pass try: resp = base64.b64decode(resp) except binascii.Error: pass if resp == 'Not Found' or not resp: abort(404) return Response(resp, mimetype=mime_type)
def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False): if not stringvalidators.validate_hash(bHash): raise onionrexceptions.InvalidHexHash( "block hash not valid hash format") bl = onionrblockapi.Block(bHash) if decrypt: bl.decrypt() if bl.isEncrypted and not bl.decrypted: raise ValueError if not raw: if not headerOnly: retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent} for x in list(retData.keys()): try: retData[x] = retData[x].decode() except AttributeError: pass else: validSig = False signer = bytesconverter.bytes_to_str(bl.signer) if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer): validSig = True bl.bheader['validSig'] = validSig bl.bheader['meta'] = '' retData = {'meta': bl.bheader, 'metadata': bl.bmetadata} return json.dumps(retData) else: return bl.raw
def getBlockBodyData(name): resp = '' if stringvalidators.validate_hash(name): try: resp = onionrblockapi.Block(name, decrypt=True).bcontent except TypeError: pass else: abort(404) return Response(resp)
def process_block_metadata(blockHash: str): ''' Read metadata from a block and cache it to the block database blockHash -> sha3_256 hex formatted hash of Onionr block ''' curTime = epoch.get_rounded_epoch(roundS=60) myBlock = onionrblockapi.Block(blockHash) if myBlock.isEncrypted: myBlock.decrypt() if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted): blockType = myBlock.getMetadata( 'type' ) # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = bytesconverter.bytes_to_str(myBlock.signer) valid = myBlock.verifySig() if valid: if myBlock.getMetadata('newFSKey') is not None: try: onionrusers.OnionrUser(signer).addForwardKey( myBlock.getMetadata('newFSKey')) except onionrexceptions.InvalidPubkey: logger.warn( '%s has invalid forward secrecy key to add: %s' % (signer, myBlock.getMetadata('newFSKey'))) try: if len(blockType) <= onionrvalues.MAX_BLOCK_TYPE_LENGTH: blockmetadb.update_block_info(blockHash, 'dataType', blockType) except TypeError: logger.warn("Missing block information") pass # Set block expire time if specified try: expireTime = int(myBlock.getHeader('expire')) # test that expire time is an integer of sane length (for epoch) # doesn't matter if its too large because of the min() func below if not len(str(expireTime)) < 20: raise ValueError('timestamp invalid') except (ValueError, TypeError) as e: expireTime = onionrvalues.DEFAULT_EXPIRE + curTime finally: expireTime = min(expireTime, curTime + onionrvalues.DEFAULT_EXPIRE) blockmetadb.update_block_info(blockHash, 'expire', expireTime) if blockType == 'update': updater.update_event(myBlock) onionrevents.event('processblocks', data={ 'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid })
def find_site_gzip(user_id: str) -> tarfile.TarFile: """Return verified site tar object""" sites = blockmetadb.get_blocks_by_type('osite') user_site = None user_id = unpaddedbase32.repad(user_id) for site in sites: block = onionrblockapi.Block(site) if block.isSigner(user_id): user_site = block if not user_site is None: return tarfile.open(fileobj=io.BytesIO(user_site.bcontent), mode='r') return None
def block_mixer(upload_list: List[onionrtypes.BlockHash], block_to_mix: onionrtypes.BlockHash): """Delay and mix block inserts. Take a block list and a received/created block and add it to the said block list """ bl = onionrblockapi.Block(block_to_mix) if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: raise ValueError if block_to_mix: upload_list.append(block_to_mix)
def load_inbox(): inbox_list = [] deleted = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/mailcache.dat').get('deleted_mail') if deleted is None: deleted = [] for blockHash in blockmetadb.get_blocks_by_type('pm'): block = onionrblockapi.Block(blockHash) block.decrypt() if block.decrypted and reconstructhash.deconstruct_hash( blockHash) not in deleted: inbox_list.append(blockHash) return inbox_list
def test_lan_server(testmanager): if onionrvalues.IS_QUBES: logger.warn("Cannot test LAN on QubesOS", terminal=True) return start_time = get_epoch() for i in range(1337, 1340): try: if not best_ip or not best_ip.startswith(('192.168')): logger.warn( "lanservertest not running, not in standard 192.168 lan " + "run this test on a lan before release", terminal=True) return if requests.get(f"http://{best_ip}:{i}/ping").text == 'onionr!': bl = insert('test data') sleep(10) bl2 = insert('test data2') sleep(30) bl3 = insert('test data3') l = requests.get(f"http://{best_ip}:{i}/blist/0").text.split( '\n') if bl not in l or bl2 not in l or bl3 not in l: logger.error('blocks not in blist ' + '-'.join(l)) raise ValueError time = blockmetadb.get_block_date(bl3) - 1 l = requests.get( f"http://{best_ip}:{i}/blist/{time}").text.split('\n') if (bl in l and bl2 in l and bl3 in l) or len(l) == 0: logger.error('Failed to get appopriate time' + '-'.join(l)) raise ValueError if onionrblockapi.Block(bl).raw != requests.get( f"http://{best_ip}:{i}/get/{bl}", stream=True).raw.read(6000000): logger.error('Block doesn\'t match') raise ValueError break except requests.exceptions.ConnectionError: pass else: raise ValueError
def service_creator(daemon): assert isinstance(daemon, communicator.OnionrCommunicatorDaemon) # Find socket connection blocks # TODO cache blocks and only look at recently received ones con_blocks = blockmetadb.get_blocks_by_type('con') for b in con_blocks: if not b in daemon.active_services: bl = onionrblockapi.Block(b, decrypt=True) bs = bytesconverter.bytes_to_str(bl.bcontent) + '.onion' if server_exists(bl.signer): continue if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs): signer = bytesconverter.bytes_to_str(bl.signer) daemon.active_services.append(b) daemon.active_services.append(signer) if not daemon.services.create_server(signer, bs, daemon): daemon.active_services.remove(b) daemon.active_services.remove(signer) daemon.decrementThreadCount('service_creator')
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): """Accept a communicator instance + upload blocks from its upload queue.""" """when inserting a block, we try to upload it to a few peers to add some deniability & increase functionality""" kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") TIMER_NAME = "upload_blocks_from_communicator" session_manager: sessionmanager.BlockUploadSessionManager session_manager = comm_inst.shared_state.get( sessionmanager.BlockUploadSessionManager) tried_peers: UserID = [] finishedUploads = [] kv.put('blocksToUpload', onionrcrypto.cryptoutils.random_shuffle(kv.get('blocksToUpload'))) def remove_from_hidden(bl): sleep(60) try: comm_inst.shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(bl) except ValueError: pass if len(kv.get('blocksToUpload')) != 0: for bl in kv.get('blocksToUpload'): if not stringvalidators.validate_hash(bl): logger.warn('Requested to upload invalid block', terminal=True) comm_inst.decrementThreadCount(TIMER_NAME) return session = session_manager.add_session(bl) for _ in range(min(len(kv.get('onlinePeers')), 6)): try: peer = onlinepeers.pick_online_peer(comm_inst) except onionrexceptions.OnlinePeerNeeded: continue try: session.peer_exists[peer] continue except KeyError: pass try: if session.peer_fails[peer] > 3: continue except KeyError: pass if peer in tried_peers: continue tried_peers.append(peer) url = f'http://{peer}/upload' try: data = block.Block(bl).getRaw() except onionrexceptions.NoDataAvailable: finishedUploads.append(bl) break proxy_type = proxypicker.pick_proxy(peer) logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True) resp = basicrequests.do_post_request( url, data=data, proxyType=proxy_type, content_type='application/octet-stream') if resp is not False: if resp == 'success': Thread(target=remove_from_hidden, args=[bl], daemon=True).start() session.success() session.peer_exists[peer] = True elif resp == 'exists': session.success() session.peer_exists[peer] = True else: session.fail() session.fail_peer(peer) comm_inst.getPeerProfileInstance(peer).addScore(-5) logger.warn( f'Failed to upload {bl[:8]}, reason: {resp}', terminal=True) else: session.fail() session_manager.clean_session() for x in finishedUploads: try: kv.get('blocksToUpload').remove(x) comm_inst.shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(x) except ValueError: pass comm_inst.decrementThreadCount(TIMER_NAME)
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon): """Accepts a communicator instance and uploads blocks from its upload queue""" """when inserting a block, we try to upload it to a few peers to add some deniability & increase functionality""" TIMER_NAME = "upload_blocks_from_communicator" session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get( sessionmanager.BlockUploadSessionManager) triedPeers = [] finishedUploads = [] comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle( comm_inst.blocksToUpload) if len(comm_inst.blocksToUpload) != 0: for bl in comm_inst.blocksToUpload: if not stringvalidators.validate_hash(bl): logger.warn('Requested to upload invalid block', terminal=True) comm_inst.decrementThreadCount(TIMER_NAME) return session = session_manager.add_session(bl) for i in range(min(len(comm_inst.onlinePeers), 6)): peer = onlinepeers.pick_online_peer(comm_inst) try: session.peer_exists[peer] continue except KeyError: pass try: if session.peer_fails[peer] > 3: continue except KeyError: pass if peer in triedPeers: continue triedPeers.append(peer) url = f'http://{peer}/upload' try: data = block.Block(bl).getRaw() except onionrexceptions.NoDataAvailable: finishedUploads.append(bl) break proxyType = proxypicker.pick_proxy(peer) logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True) resp = basicrequests.do_post_request( url, data=data, proxyType=proxyType, content_type='application/octet-stream') if not resp == False: if resp == 'success': session.success() session.peer_exists[peer] = True elif resp == 'exists': session.success() session.peer_exists[peer] = True else: session.fail() session.fail_peer(peer) comm_inst.getPeerProfileInstance(peer).addScore(-5) logger.warn( f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True) else: session.fail() session_manager.clean_session() for x in finishedUploads: try: comm_inst.blocksToUpload.remove(x) except ValueError: pass comm_inst.decrementThreadCount(TIMER_NAME)
def test_encrypted_insert(self): setup_test() message = 'hello world2' bl = onionrblocks.insert(message, asymPeer=onionrcrypto.pub_key) self.assertIn(bytesconverter.str_to_bytes(message), onionrblockapi.Block(bl, decrypt=True).bcontent)
def test_time_insert_10(self): bl = time_insert('test', delay=10) self.assertTrue(bl) bl = onionrblockapi.Block(bl) self.assertIs(bl.bmetadata['dly'], 10)
def test_time_insert_none(self): bl = time_insert('test') self.assertTrue(bl) bl = onionrblockapi.Block(bl) self.assertIs(bl.bmetadata['dly'], 0)
def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'): """Accept a communicator instance + upload blocks from its upload queue.""" """when inserting a block, we try to upload it to a few peers to add some deniability & increase functionality""" kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") session_manager: sessionmanager.BlockUploadSessionManager session_manager = shared_state.get( sessionmanager.BlockUploadSessionManager) tried_peers: UserID = [] finishedUploads = [] SystemRandom().shuffle(kv.get('blocksToUpload')) def remove_from_hidden(bl): sleep(60) try: shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(bl) except ValueError: pass if len(kv.get('blocksToUpload')) != 0: for bl in kv.get('blocksToUpload'): if not stringvalidators.validate_hash(bl): logger.warn('Requested to upload invalid block', terminal=True) return session = session_manager.add_session(bl) for _ in range(min(len(kv.get('onlinePeers')), 6)): try: peer = onlinepeers.pick_online_peer(kv) if not block.Block(bl).isEncrypted: if peer in kv.get('plaintextDisabledPeers'): logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa continue except onionrexceptions.OnlinePeerNeeded: continue try: session.peer_exists[peer] continue except KeyError: pass try: if session.peer_fails[peer] > 3: continue except KeyError: pass if peer in tried_peers: continue tried_peers.append(peer) url = f'http://{peer}/upload' try: data = block.Block(bl).getRaw() if not data: logger.warn( f"Couldn't get data for block in upload list {bl}", terminal=True) raise onionrexceptions.NoDataAvailable try: def __check_metadata(): metadata = get_block_metadata_from_data(data)[0] if not validate_metadata(metadata, data): logger.warn( f"Metadata for uploading block not valid {bl}") raise onionrexceptions.InvalidMetadata __check_metadata() except onionrexceptions.DataExists: pass except( # noqa onionrexceptions.NoDataAvailable, onionrexceptions.InvalidMetadata) as _: finishedUploads.append(bl) break proxy_type = proxypicker.pick_proxy(peer) logger.info( f"Uploading block {bl[:8]} to {peer}", terminal=True) resp = basicrequests.do_post_request( url, data=data, proxyType=proxy_type, content_type='application/octet-stream') if resp is not False: if resp == 'success': Thread(target=remove_from_hidden, args=[bl], daemon=True).start() session.success() session.peer_exists[peer] = True elif resp == 'exists': session.success() session.peer_exists[peer] = True else: session.fail() session.fail_peer(peer) shared_state.get_by_string( 'OnionrCommunicatorDaemon').getPeerProfileInstance( peer).addScore(-5) logger.warn( f'Failed to upload {bl[:8]}, reason: {resp}', terminal=True) else: session.fail() session_manager.clean_session() for x in finishedUploads: try: kv.get('blocksToUpload').remove(x) shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(x) except ValueError: pass