def on_processblocks(api, data=None): metadata = data['block'].bmetadata # Get the block metadata if data['type'] != 'brd': return b_hash = reconstructhash.deconstruct_hash(data['block'].hash) # Get the 0-truncated block hash board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False) # get the board index cache board_cache.refresh() # Validate the channel name is sane for caching try: ch = metadata['ch'] except KeyError: ch = 'global' ch_len = len(ch) if ch_len == 0: ch = 'global' elif ch_len > 12: return existing_posts = board_cache.get(ch) if existing_posts is None: existing_posts = [] existing_posts.append(data['block'].hash) board_cache.put(ch, existing_posts) board_cache.flush()
def __init__(self, client_api): direct_conn_management_bp = Blueprint('direct_conn_management', __name__) self.direct_conn_management_bp = direct_conn_management_bp cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage) @direct_conn_management_bp.route('/dc-client/isconnected/<pubkey>') def is_connected(pubkey): communicator = _get_communicator(g) resp = "" if pubkey in communicator.direct_connection_clients: resp = communicator.direct_connection_clients[pubkey] return Response(resp) @direct_conn_management_bp.route('/dc-client/connect/<pubkey>') def make_new_connection(pubkey): communicator = _get_communicator(g) resp = "pending" if pubkey in communicator.shared_state.get( pool.ServicePool).bootstrap_pending: return Response(resp) if pubkey in communicator.direct_connection_clients: resp = communicator.direct_connection_clients[pubkey] else: """Spawn a thread that will create the client and eventually add it to the communicator.active_services """ threading.Thread( target=onionrservices.OnionrServices().create_client, args=[pubkey, communicator], daemon=True).start() return Response(resp)
def handle_announce(request): """accept announcement posts, validating POW clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request """ resp = 'failure' newNode = '' try: newNode = request.form['node'].encode() except KeyError: logger.warn('No node specified for upload') else: newNode = bytesconverter.bytes_to_str(newNode) announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache) announce_queue_list = announce_queue.get('new_peers') if announce_queue_list is None: announce_queue_list = [] else: if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE: newNode = '' if stringvalidators.validate_transport(newNode) and \ newNode not in announce_queue_list: g.shared_state.get( deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode) announce_queue.put('new_peers', announce_queue_list.append(newNode)) announce_queue.flush() resp = 'Success' resp = Response(resp) if resp == 'failure': return resp, 406 return resp
def get_post_by_board(board): board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False) board_cache.refresh() posts = board_cache.get(board) if posts is None: posts = '' else: posts = ','.join(posts) return Response(posts)
def handle_announce(request): ''' accept announcement posts, validating POW clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request ''' resp = 'failure' powHash = '' randomData = '' newNode = '' try: newNode = request.form['node'].encode() except KeyError: logger.warn('No node specified for upload') pass else: try: randomData = request.form['random'] randomData = base64.b64decode(randomData) except KeyError: logger.warn('No random data specified for upload') else: nodes = newNode + bytesconverter.str_to_bytes( gettransports.get()[0]) nodes = crypto.hashers.blake2b_hash(nodes) powHash = crypto.hashers.blake2b_hash(randomData + nodes) try: powHash = powHash.decode() except AttributeError: pass if powHash.startswith('0' * onionrvalues.ANNOUNCE_POW): newNode = bytesconverter.bytes_to_str(newNode) announce_queue = deadsimplekv.DeadSimpleKV( filepaths.announce_cache) announce_queue_list = announce_queue.get('new_peers') if announce_queue_list is None: announce_queue_list = [] if stringvalidators.validate_transport( newNode) and not newNode in announce_queue_list: #clientAPI.onionrInst.communicatorInst.newPeers.append(newNode) g.shared_state.get( OnionrCommunicatorDaemon).newPeers.append(newNode) announce_queue.put('new_peers', announce_queue_list.append(newNode)) announce_queue.flush() resp = 'Success' else: logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) resp = Response(resp) if resp == 'failure': return resp, 406 return resp
def get_post_by_board_with_offset(board, offset): offset = int(offset) OFFSET_COUNT = 10 board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False) board_cache.refresh() posts = board_cache.get(board) if posts is None: posts = '' else: posts.reverse() posts = ','.join(posts[offset:offset + OFFSET_COUNT]) return Response(posts)
def load_inbox(): inbox_list = [] deleted = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/mailcache.dat').get('deleted_mail') if deleted is None: deleted = [] for blockHash in blockmetadb.get_blocks_by_type('pm'): block = onionrblockapi.Block(blockHash) block.decrypt() if block.decrypted and reconstructhash.deconstruct_hash( blockHash) not in deleted: inbox_list.append(blockHash) return inbox_list
def __init__(self, communicator: 'OnionrCommunicatorDaemon'): """Start the UploadQueue object, loading left over uploads into queue and registering save shutdown function """ self.communicator = communicator cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE) self.store_obj = cache cache: list = cache.get('uploads') if cache == None: cache = [] _add_to_hidden_blocks(cache) self.communicator.blocksToUpload.extend(cache) atexit.register(self.save)
def __init__(self, communicator: 'OnionrCommunicatorDaemon'): """Start the UploadQueue object, loading left over uploads into queue. register save shutdown function """ self.communicator = communicator cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV( UPLOAD_MEMORY_FILE) self.kv: "DeadSimpleKV" = communicator.shared_state.get_by_string( "DeadSimpleKV") self.store_obj = cache cache = cache.get('uploads') if cache is None: cache = [] _add_to_hidden_blocks(cache) self.kv.get('blocksToUpload').extend(cache) atexit.register(self.save)
def __init__(self, torPort=0): ''' Initialize Core Onionr library ''' # set data dir self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/')) if not self.dataDir.endswith('/'): self.dataDir += '/' try: self.onionrInst = None self.queueDB = self.dataDir + 'queue.db' self.peerDB = self.dataDir + 'peers.db' self.blockDB = self.dataDir + 'blocks.db' self.blockDataLocation = self.dataDir + 'blocks/' self.blockDataDB = self.blockDataLocation + 'block-data.db' self.publicApiHostFile = self.dataDir + 'public-host.txt' self.privateApiHostFile = self.dataDir + 'private-host.txt' self.addressDB = self.dataDir + 'address.db' self.hsAddress = '' self.i2pAddress = config.get('i2p.own_addr', None) self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() self.torPort = torPort self.dataNonceFile = self.dataDir + 'block-nonces.dat' self.dbCreate = dbcreator.DBCreator(self) self.forwardKeysFile = self.dataDir + 'forward-keys.db' self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5) # Socket data, defined here because of multithreading constraints with gevent self.killSockets = False self.startSocket = {} self.socketServerConnData = {} self.socketReasons = {} self.socketServerResponseData = {} self.usageFile = self.dataDir + 'disk-usage.txt' self.config = config self.maxBlockSize = 10000000 # max block size in bytes if not os.path.exists(self.dataDir): os.mkdir(self.dataDir) if not os.path.exists(self.dataDir + 'blocks/'): os.mkdir(self.dataDir + 'blocks/') if not os.path.exists(self.blockDB): self.createBlockDB() if not os.path.exists(self.forwardKeysFile): self.dbCreate.createForwardKeyDB() if not os.path.exists(self.peerDB): self.createPeerDB() if not os.path.exists(self.addressDB): self.createAddressDB() if os.path.exists(self.dataDir + '/hs/hostname'): with open(self.dataDir + '/hs/hostname', 'r') as hs: self.hsAddress = hs.read().strip() # Load bootstrap address list if os.path.exists(self.bootstrapFileLocation): with open(self.bootstrapFileLocation, 'r') as bootstrap: bootstrap = bootstrap.read() for i in bootstrap.split('\n'): self.bootstrapList.append(i) else: logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation) self.use_subprocess = powchoice.use_subprocess(self) self._utils = onionrutils.OnionrUtils(self) # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) self._blacklist = onionrblacklist.OnionrBlackList(self) self.serializer = serializeddata.SerializedData(self) except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) logger.fatal('Cannot recover from error.') sys.exit(1) return
You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. ''' import sys, os, json from flask import Response, request, redirect, Blueprint, abort from onionrusers import contactmanager from onionrutils import stringvalidators from utils import reconstructhash, identifyhome import filepaths import deadsimplekv as simplekv sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) import loadinbox, sentboxdb flask_blueprint = Blueprint('mail', __name__) kv = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/mailcache.dat') @flask_blueprint.route('/mail/ping') def mail_ping(): return 'pong!' @flask_blueprint.route('/mail/deletemsg/<block>', methods=['POST']) def mail_delete(block): if not stringvalidators.validate_hash(block): abort(504) block = reconstructhash.deconstruct_hash(block) existing = kv.get('deleted_mail') if existing is None: existing = []
def __init__(self, peer, address, comm_inst=None): if not stringvalidators.validate_pub_key(peer): raise ValueError('Peer must be valid base32 ed25519 public key') socks = config.get( 'tor.socksport') # Load config for Tor socks port for proxy service_app = Flask(__name__) # Setup Flask app for server. service_port = get_open_port() service_ip = apiutils.setbindip.set_bind_IP() http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) comm_inst.service_greenlets.append(http_server) key_store = simplekv.DeadSimpleKV(filepaths.cached_storage) # TODO define basic endpoints useful for direct connections like stats httpapi.load_plugin_blueprints(service_app, blueprint='direct_blueprint') @service_app.route('/ping') def get_ping(): return "pong!" @service_app.route('/close') def shutdown_server(): comm_inst.service_greenlets.remove(http_server) http_server.stop() return Response('goodbye') @service_app.after_request def afterReq(resp): # Security headers resp = httpheaders.set_default_onionr_http_headers(resp) return resp with Controller.from_port( port=config.get('tor.controlPort')) as controller: # Connect to the Tor process for Onionr controller.authenticate(config.get('tor.controlpassword')) # Create the v3 onion service for the peer to connect to response = controller.create_ephemeral_hidden_service( {80: service_port}, await_publication=True, key_type='NEW', key_content='ED25519-V3') try: for x in range(3): attempt = basicrequests.do_post_request( 'http://' + address + '/bs/' + response.service_id, port=socks) if attempt == 'success': break else: raise ConnectionError except ConnectionError: # Re-raise raise ConnectionError( 'Could not reach %s bootstrap address %s' % (peer, address)) else: # If no connection error, create the service and save it to local global key store peer = bytesconverter.bytes_to_str(peer) key_store.put('dc-' + peer, response.service_id) key_store.flush() logger.info('hosting on %s with %s' % (response.service_id, peer)) http_server.serve_forever() http_server.stop() key_store.delete('dc-' + peer)
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. ''' import urllib, requests, time import logger, config, deadsimplekv from . import getclientapiserver import filepaths config.reload() cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=1000) def get_hostname(): hostname = '' waited = 0 maxWait = 3 while True: if cache.get('client_api') is None: try: hostname = getclientapiserver.get_client_API_server() except FileNotFoundError: hostname = False else: cache.put('hostname', hostname) cache.flush() else: hostname = cache.get('hostname')
it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. """ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) direct_blueprint = Blueprint('chat', __name__) key_store = simplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=5) storage_dir = identifyhome.identify_home() @direct_blueprint.before_request def request_setup(): key_store.refresh() host = request.host host = host.strip('.b32.i2p') host = host.strip('.onion') g.host = host g.peer = key_store.get('dc-' + g.host) @direct_blueprint.route('/chat/ping') def pingdirect():
def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300): ''' Bootstrap client services ''' if not stringvalidators.validate_pub_key(peer): raise ValueError('Peer must be valid base32 ed25519 public key') connection_pool = None # here we use a lambda for the timeout thread to set to true timed_out = lambda: None timed_out.timed_out = False bootstrap_port = get_open_port() bootstrap_app = Flask(__name__) bootstrap_app.config['MAX_CONTENT_LENGTH'] = 1 * 1024 http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) try: if comm_inst is None: raise ValueError except (AttributeError, ValueError) as e: pass else: comm_inst.service_greenlets.append(http_server) connection_pool = comm_inst.shared_state.get(pool.ServicePool) bootstrap_address = '' shutdown = False bs_id = str(uuid.uuid4()) key_store = simplekv.DeadSimpleKV(filepaths.cached_storage) @bootstrap_app.route('/ping') def get_ping(): return "pong!" @bootstrap_app.after_request def afterReq(resp): # Security headers resp = httpheaders.set_default_onionr_http_headers(resp) return resp @bootstrap_app.route('/bs/<address>', methods=['POST']) def get_bootstrap(address): if stringvalidators.validate_transport(address + '.onion'): # Set the bootstrap address then close the server bootstrap_address = address + '.onion' key_store.put(bs_id, bootstrap_address) http_server.stop() return Response("success") else: return Response("") with Controller.from_port( port=config.get('tor.controlPort')) as controller: if not connection_pool is None: connection_pool.bootstrap_pending.append(peer) # Connect to the Tor process for Onionr controller.authenticate(config.get('tor.controlpassword')) # Create the v3 onion service response = controller.create_ephemeral_hidden_service( {80: bootstrap_port}, key_type='NEW', key_content='ED25519-V3', await_publication=True) onionrblocks.insert(response.service_id, header='con', sign=True, encryptType='asym', asymPeer=peer, disableForward=True, expire=(epoch.get_epoch() + bootstrap_timeout)) threading.Thread(target=__bootstrap_timeout, args=[http_server, bootstrap_timeout, timed_out], daemon=True).start() # Run the bootstrap server try: http_server.serve_forever() except TypeError: pass # This line reached when server is shutdown by being bootstrapped # Add the address to the client pool if not comm_inst is None: connection_pool.bootstrap_pending.remove(peer) if timed_out.timed_out: logger.warn('Could not connect to %s due to timeout' % (peer, )) return None comm_inst.direct_connection_clients[peer] = response.service_id # Now that the bootstrap server has received a server, return the address return key_store.get(bs_id)
def server_exists(peer: str) -> bool: '''checks if an onion server is created for a peer or not''' peer = bytesconverter.bytes_to_str(peer) kv = deadsimplekv.DeadSimpleKV(filepaths.cached_storage) kv.refresh() return not kv.get('dc-' + peer) is None