def on_processblocks(api, data=None):
metadata = data['block'].bmetadata # Get the block metadata
if data['type'] != 'brd':
return
b_hash = reconstructhash.deconstruct_hash(data['block'].hash) # Get the 0-truncated block hash
board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False) # get the board index cache
board_cache.refresh()
# Validate the channel name is sane for caching
try:
ch = metadata['ch']
except KeyError:
ch = 'global'
ch_len = len(ch)
if ch_len == 0:
ch = 'global'
elif ch_len > 12:
return
existing_posts = board_cache.get(ch)
if existing_posts is None:
existing_posts = []
existing_posts.append(data['block'].hash)
board_cache.put(ch, existing_posts)
board_cache.flush()
def __init__(self, client_api):
direct_conn_management_bp = Blueprint('direct_conn_management',
__name__)
self.direct_conn_management_bp = direct_conn_management_bp
cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage)
@direct_conn_management_bp.route('/dc-client/isconnected/<pubkey>')
def is_connected(pubkey):
communicator = _get_communicator(g)
resp = ""
if pubkey in communicator.direct_connection_clients:
resp = communicator.direct_connection_clients[pubkey]
return Response(resp)
@direct_conn_management_bp.route('/dc-client/connect/<pubkey>')
def make_new_connection(pubkey):
communicator = _get_communicator(g)
resp = "pending"
if pubkey in communicator.shared_state.get(
pool.ServicePool).bootstrap_pending:
return Response(resp)
if pubkey in communicator.direct_connection_clients:
resp = communicator.direct_connection_clients[pubkey]
else:
"""Spawn a thread that will create the client and eventually add it to the
communicator.active_services
"""
threading.Thread(
target=onionrservices.OnionrServices().create_client,
args=[pubkey, communicator],
daemon=True).start()
return Response(resp)
def handle_announce(request):
"""accept announcement posts, validating POW
clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
"""
resp = 'failure'
newNode = ''
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No node specified for upload')
else:
newNode = bytesconverter.bytes_to_str(newNode)
announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
announce_queue_list = announce_queue.get('new_peers')
if announce_queue_list is None:
announce_queue_list = []
else:
if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE:
newNode = ''
if stringvalidators.validate_transport(newNode) and \
newNode not in announce_queue_list:
g.shared_state.get(
deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode)
announce_queue.put('new_peers',
announce_queue_list.append(newNode))
announce_queue.flush()
resp = 'Success'
resp = Response(resp)
if resp == 'failure':
return resp, 406
return resp
def get_post_by_board(board):
board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False)
board_cache.refresh()
posts = board_cache.get(board)
if posts is None:
posts = ''
else:
posts = ','.join(posts)
return Response(posts)
def handle_announce(request):
'''
accept announcement posts, validating POW
clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
'''
resp = 'failure'
powHash = ''
randomData = ''
newNode = ''
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No node specified for upload')
pass
else:
try:
randomData = request.form['random']
randomData = base64.b64decode(randomData)
except KeyError:
logger.warn('No random data specified for upload')
else:
nodes = newNode + bytesconverter.str_to_bytes(
gettransports.get()[0])
nodes = crypto.hashers.blake2b_hash(nodes)
powHash = crypto.hashers.blake2b_hash(randomData + nodes)
try:
powHash = powHash.decode()
except AttributeError:
pass
if powHash.startswith('0' * onionrvalues.ANNOUNCE_POW):
newNode = bytesconverter.bytes_to_str(newNode)
announce_queue = deadsimplekv.DeadSimpleKV(
filepaths.announce_cache)
announce_queue_list = announce_queue.get('new_peers')
if announce_queue_list is None:
announce_queue_list = []
if stringvalidators.validate_transport(
newNode) and not newNode in announce_queue_list:
#clientAPI.onionrInst.communicatorInst.newPeers.append(newNode)
g.shared_state.get(
OnionrCommunicatorDaemon).newPeers.append(newNode)
announce_queue.put('new_peers',
announce_queue_list.append(newNode))
announce_queue.flush()
resp = 'Success'
else:
logger.warn(newNode.decode() + ' failed to meet POW: ' +
powHash)
resp = Response(resp)
if resp == 'failure':
return resp, 406
return resp
def get_post_by_board_with_offset(board, offset):
offset = int(offset)
OFFSET_COUNT = 10
board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False)
board_cache.refresh()
posts = board_cache.get(board)
if posts is None:
posts = ''
else:
posts.reverse()
posts = ','.join(posts[offset:offset + OFFSET_COUNT])
return Response(posts)
def load_inbox():
inbox_list = []
deleted = simplekv.DeadSimpleKV(identifyhome.identify_home() +
'/mailcache.dat').get('deleted_mail')
if deleted is None:
deleted = []
for blockHash in blockmetadb.get_blocks_by_type('pm'):
block = onionrblockapi.Block(blockHash)
block.decrypt()
if block.decrypted and reconstructhash.deconstruct_hash(
blockHash) not in deleted:
inbox_list.append(blockHash)
return inbox_list
def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
"""Start the UploadQueue object, loading left over uploads into queue
and registering save shutdown function
"""
self.communicator = communicator
cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE)
self.store_obj = cache
cache: list = cache.get('uploads')
if cache == None:
cache = []
_add_to_hidden_blocks(cache)
self.communicator.blocksToUpload.extend(cache)
atexit.register(self.save)
def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
"""Start the UploadQueue object, loading left over uploads into queue.
register save shutdown function
"""
self.communicator = communicator
cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV(
UPLOAD_MEMORY_FILE)
self.kv: "DeadSimpleKV" = communicator.shared_state.get_by_string(
"DeadSimpleKV")
self.store_obj = cache
cache = cache.get('uploads')
if cache is None:
cache = []
_add_to_hidden_blocks(cache)
self.kv.get('blocksToUpload').extend(cache)
atexit.register(self.save)
def __init__(self, torPort=0):
'''
Initialize Core Onionr library
'''
# set data dir
self.dataDir = os.environ.get('ONIONR_HOME',
os.environ.get('DATA_DIR', 'data/'))
if not self.dataDir.endswith('/'):
self.dataDir += '/'
try:
self.onionrInst = None
self.queueDB = self.dataDir + 'queue.db'
self.peerDB = self.dataDir + 'peers.db'
self.blockDB = self.dataDir + 'blocks.db'
self.blockDataLocation = self.dataDir + 'blocks/'
self.blockDataDB = self.blockDataLocation + 'block-data.db'
self.publicApiHostFile = self.dataDir + 'public-host.txt'
self.privateApiHostFile = self.dataDir + 'private-host.txt'
self.addressDB = self.dataDir + 'address.db'
self.hsAddress = ''
self.i2pAddress = config.get('i2p.own_addr', None)
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.dataNonceFile = self.dataDir + 'block-nonces.dat'
self.dbCreate = dbcreator.DBCreator(self)
self.forwardKeysFile = self.dataDir + 'forward-keys.db'
self.keyStore = simplekv.DeadSimpleKV(self.dataDir +
'cachedstorage.dat',
refresh_seconds=5)
# Socket data, defined here because of multithreading constraints with gevent
self.killSockets = False
self.startSocket = {}
self.socketServerConnData = {}
self.socketReasons = {}
self.socketServerResponseData = {}
self.usageFile = self.dataDir + 'disk-usage.txt'
self.config = config
self.maxBlockSize = 10000000 # max block size in bytes
if not os.path.exists(self.dataDir):
os.mkdir(self.dataDir)
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')
if not os.path.exists(self.blockDB):
self.createBlockDB()
if not os.path.exists(self.forwardKeysFile):
self.dbCreate.createForwardKeyDB()
if not os.path.exists(self.peerDB):
self.createPeerDB()
if not os.path.exists(self.addressDB):
self.createAddressDB()
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()
# Load bootstrap address list
if os.path.exists(self.bootstrapFileLocation):
with open(self.bootstrapFileLocation, 'r') as bootstrap:
bootstrap = bootstrap.read()
for i in bootstrap.split('\n'):
self.bootstrapList.append(i)
else:
logger.warn('Warning: address bootstrap file not found ' +
self.bootstrapFileLocation)
self.use_subprocess = powchoice.use_subprocess(self)
self._utils = onionrutils.OnionrUtils(self)
# Initialize the crypto object
self._crypto = onionrcrypto.OnionrCrypto(self)
self._blacklist = onionrblacklist.OnionrBlackList(self)
self.serializer = serializeddata.SerializedData(self)
except Exception as error:
logger.error('Failed to initialize core Onionr library.',
error=error)
logger.fatal('Cannot recover from error.')
sys.exit(1)
return
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sys, os, json
from flask import Response, request, redirect, Blueprint, abort
from onionrusers import contactmanager
from onionrutils import stringvalidators
from utils import reconstructhash, identifyhome
import filepaths
import deadsimplekv as simplekv
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import loadinbox, sentboxdb
flask_blueprint = Blueprint('mail', __name__)
kv = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/mailcache.dat')
@flask_blueprint.route('/mail/ping')
def mail_ping():
return 'pong!'
@flask_blueprint.route('/mail/deletemsg/<block>', methods=['POST'])
def mail_delete(block):
if not stringvalidators.validate_hash(block):
abort(504)
block = reconstructhash.deconstruct_hash(block)
existing = kv.get('deleted_mail')
if existing is None:
existing = []
def __init__(self, peer, address, comm_inst=None):
if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key')
socks = config.get(
'tor.socksport') # Load config for Tor socks port for proxy
service_app = Flask(__name__) # Setup Flask app for server.
service_port = get_open_port()
service_ip = apiutils.setbindip.set_bind_IP()
http_server = WSGIServer(('127.0.0.1', service_port),
service_app,
log=None)
comm_inst.service_greenlets.append(http_server)
key_store = simplekv.DeadSimpleKV(filepaths.cached_storage)
# TODO define basic endpoints useful for direct connections like stats
httpapi.load_plugin_blueprints(service_app,
blueprint='direct_blueprint')
@service_app.route('/ping')
def get_ping():
return "pong!"
@service_app.route('/close')
def shutdown_server():
comm_inst.service_greenlets.remove(http_server)
http_server.stop()
return Response('goodbye')
@service_app.after_request
def afterReq(resp):
# Security headers
resp = httpheaders.set_default_onionr_http_headers(resp)
return resp
with Controller.from_port(
port=config.get('tor.controlPort')) as controller:
# Connect to the Tor process for Onionr
controller.authenticate(config.get('tor.controlpassword'))
# Create the v3 onion service for the peer to connect to
response = controller.create_ephemeral_hidden_service(
{80: service_port},
await_publication=True,
key_type='NEW',
key_content='ED25519-V3')
try:
for x in range(3):
attempt = basicrequests.do_post_request(
'http://' + address + '/bs/' + response.service_id,
port=socks)
if attempt == 'success':
break
else:
raise ConnectionError
except ConnectionError:
# Re-raise
raise ConnectionError(
'Could not reach %s bootstrap address %s' %
(peer, address))
else:
# If no connection error, create the service and save it to local global key store
peer = bytesconverter.bytes_to_str(peer)
key_store.put('dc-' + peer, response.service_id)
key_store.flush()
logger.info('hosting on %s with %s' %
(response.service_id, peer))
http_server.serve_forever()
http_server.stop()
key_store.delete('dc-' + peer)
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import urllib, requests, time
import logger, config, deadsimplekv
from . import getclientapiserver
import filepaths
config.reload()
cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=1000)
def get_hostname():
hostname = ''
waited = 0
maxWait = 3
while True:
if cache.get('client_api') is None:
try:
hostname = getclientapiserver.get_client_API_server()
except FileNotFoundError:
hostname = False
else:
cache.put('hostname', hostname)
cache.flush()
else:
hostname = cache.get('hostname')
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
direct_blueprint = Blueprint('chat', __name__)
key_store = simplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=5)
storage_dir = identifyhome.identify_home()
@direct_blueprint.before_request
def request_setup():
key_store.refresh()
host = request.host
host = host.strip('.b32.i2p')
host = host.strip('.onion')
g.host = host
g.peer = key_store.get('dc-' + g.host)
@direct_blueprint.route('/chat/ping')
def pingdirect():
def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300):
'''
Bootstrap client services
'''
if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key')
connection_pool = None
# here we use a lambda for the timeout thread to set to true
timed_out = lambda: None
timed_out.timed_out = False
bootstrap_port = get_open_port()
bootstrap_app = Flask(__name__)
bootstrap_app.config['MAX_CONTENT_LENGTH'] = 1 * 1024
http_server = WSGIServer(('127.0.0.1', bootstrap_port),
bootstrap_app,
log=None)
try:
if comm_inst is None: raise ValueError
except (AttributeError, ValueError) as e:
pass
else:
comm_inst.service_greenlets.append(http_server)
connection_pool = comm_inst.shared_state.get(pool.ServicePool)
bootstrap_address = ''
shutdown = False
bs_id = str(uuid.uuid4())
key_store = simplekv.DeadSimpleKV(filepaths.cached_storage)
@bootstrap_app.route('/ping')
def get_ping():
return "pong!"
@bootstrap_app.after_request
def afterReq(resp):
# Security headers
resp = httpheaders.set_default_onionr_http_headers(resp)
return resp
@bootstrap_app.route('/bs/<address>', methods=['POST'])
def get_bootstrap(address):
if stringvalidators.validate_transport(address + '.onion'):
# Set the bootstrap address then close the server
bootstrap_address = address + '.onion'
key_store.put(bs_id, bootstrap_address)
http_server.stop()
return Response("success")
else:
return Response("")
with Controller.from_port(
port=config.get('tor.controlPort')) as controller:
if not connection_pool is None:
connection_pool.bootstrap_pending.append(peer)
# Connect to the Tor process for Onionr
controller.authenticate(config.get('tor.controlpassword'))
# Create the v3 onion service
response = controller.create_ephemeral_hidden_service(
{80: bootstrap_port},
key_type='NEW',
key_content='ED25519-V3',
await_publication=True)
onionrblocks.insert(response.service_id,
header='con',
sign=True,
encryptType='asym',
asymPeer=peer,
disableForward=True,
expire=(epoch.get_epoch() + bootstrap_timeout))
threading.Thread(target=__bootstrap_timeout,
args=[http_server, bootstrap_timeout, timed_out],
daemon=True).start()
# Run the bootstrap server
try:
http_server.serve_forever()
except TypeError:
pass
# This line reached when server is shutdown by being bootstrapped
# Add the address to the client pool
if not comm_inst is None:
connection_pool.bootstrap_pending.remove(peer)
if timed_out.timed_out:
logger.warn('Could not connect to %s due to timeout' % (peer, ))
return None
comm_inst.direct_connection_clients[peer] = response.service_id
# Now that the bootstrap server has received a server, return the address
return key_store.get(bs_id)
def server_exists(peer: str) -> bool:
'''checks if an onion server is created for a peer or not'''
peer = bytesconverter.bytes_to_str(peer)
kv = deadsimplekv.DeadSimpleKV(filepaths.cached_storage)
kv.refresh()
return not kv.get('dc-' + peer) is None
