def detect_disk_access(info):
if type(info[0]) is int: return
if '/dev/null' == info[0]: return
whitelist = [
identify_home(), 'onionr/src/', '/site-packages/', '/usr/lib64/'
]
for item in whitelist:
if item in info[0]:
return
if identify_home() not in info[0]:
if 'proc' not in info[0]: # if it is, it is onionr stats
logger.warn(f'[DISK MINISTRY] {info}')
def get_stats(self):
"""Return statistics about our node"""
stats = {}
proc = Process()
def get_open_files():
if WINDOWS: return proc.num_handles()
return proc.num_fds()
try:
self._too_many
except AttributeError:
sleep(1)
comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon,
args=(self._too_many, ))
connected = []
[
connected.append(x) for x in comm_inst.onlinePeers
if x not in connected
]
stats['uptime'] = comm_inst.getUptime()
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(comm_inst.blockQueue)
stats['threads'] = proc.num_threads()
stats['ramPercent'] = proc.memory_percent()
stats['fd'] = get_open_files()
stats['diskUsage'] = human_size(size(identify_home()))
return json.dumps(stats)
def on_processblocks(api, data=None):
metadata = data['block'].bmetadata # Get the block metadata
if data['type'] != 'brd':
return
b_hash = reconstructhash.deconstruct_hash(data['block'].hash) # Get the 0-truncated block hash
board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False) # get the board index cache
board_cache.refresh()
# Validate the channel name is sane for caching
try:
ch = metadata['ch']
except KeyError:
ch = 'global'
ch_len = len(ch)
if ch_len == 0:
ch = 'global'
elif ch_len > 12:
return
existing_posts = board_cache.get(ch)
if existing_posts is None:
existing_posts = []
existing_posts.append(data['block'].hash)
board_cache.put(ch, existing_posts)
board_cache.flush()
def test_torrc_get(self):
torrc = identifyhome.identify_home() + '/torrc-custom'
self.assertEqual(customtorrc.get_custom_torrc(), '\n')
with open(torrc, 'w') as torrc_file:
torrc_file.write('test')
self.assertEqual(customtorrc.get_custom_torrc(), '\ntest')
os.remove(torrc)
def block_exec(event, info):
"""Prevent arbitrary code execution in eval/exec and log it."""
# because libraries have stupid amounts of compile/exec/eval,
# We have to use a whitelist where it can be tolerated
# Generally better than nothing, not a silver bullet
whitelisted_code = [
'netrc.py', 'shlex.py', 'gzip.py', '<werkzeug routing>',
'werkzeug/test.py', 'multiprocessing/popen_fork.py',
'multiprocessing/util.py', 'multiprocessing/connection.py',
'multiprocessing/queues.py', 'multiprocessing/synchronize.py',
'onionrutils/escapeansi.py', 'stem/connection.py',
'stem/response/add_onion.py', 'stem/response/authchallenge.py',
'stem/response/getinfo.py', 'stem/response/getconf.py',
'stem/response/mapaddress.py', 'stem/response/protocolinfo.py',
'apport/__init__.py', 'apport/report.py'
]
whitelisted_source = []
home = identifyhome.identify_home()
code_b64 = base64.b64encode(info[0].co_code).decode()
if code_b64 in whitelisted_source:
return
for source in whitelisted_code:
if info[0].co_filename.endswith(source):
return
if 'plugins/' in info[0].co_filename:
return
logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)
logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename)
logger.warn('Prevented exec/eval. Report this with the sample below')
logger.warn(f'{event} code in base64 format: {code_b64}')
raise ArbitraryCodeExec("Arbitrary code (eval/exec) detected.")
def get_stats(self):
"""Return statistics about our node"""
stats = {}
proc = Process()
def get_open_files():
if WINDOWS:
return proc.num_handles()
return proc.num_fds()
try:
self._too_many
except AttributeError:
sleep(1)
kv: "DeadSimpleKV" = self._too_many.get_by_string("DeadSimpleKV")
connected = []
[
connected.append(x) for x in kv.get('onlinePeers')
if x not in connected
]
stats['uptime'] = get_epoch() - kv.get('startTime')
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(kv.get('blockQueue'))
stats['threads'] = proc.num_threads()
stats['ramPercent'] = proc.memory_percent()
stats['fd'] = get_open_files()
stats['diskUsage'] = human_size(size(identify_home()))
return json.dumps(stats)
def reset():
"""Reinstalls Onionr default plugins"""
home = identifyhome.identify_home()
plugin_dir = home + '/plugins/'
if not os.path.exists(home): return
if os.path.exists(plugin_dir): shutil.rmtree(plugin_dir)
logger.info('Default plugins have been reset.', terminal=True)
def auto_refresher():
observer = Observer()
observer.schedule(Refresher(),
identify_home(),
recursive=False)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(120)
def get_post_by_board(board):
board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False)
board_cache.refresh()
posts = board_cache.get(board)
if posts is None:
posts = ''
else:
posts = ','.join(posts)
return Response(posts)
def __delete(directory):
tor_dir = '%s/%s/' % (identifyhome.identify_home(), directory)
if os.path.exists(tor_dir):
if localcommand.local_command('/ping') == 'pong!':
logger.warn('Cannot delete Tor data while Onionr is running',
terminal=True)
else:
shutil.rmtree(tor_dir)
logger.info('Tor reset', terminal=True)
def generate_torrc(net_controller: 'NetController',
api_server_ip: 'LoopBackIP'):
"""Generate a torrc file for our tor instance."""
socks_port = net_controller.socksPort
hs_port = net_controller.hsPort
home_dir = identifyhome.identify_home()
tor_config_location = home_dir + '/torrc'
hs_ver = 'HiddenServiceVersion 3'
"""
Set the Tor control password.
Meant to make it harder to manipulate our Tor instance
"""
plaintext = base64.b85encode(
os.urandom(50)).decode()
config.set('tor.controlpassword', plaintext, savefile=True)
config.set('tor.socksport', socks_port, savefile=True)
control_port = getopenport.get_open_port()
config.set('tor.controlPort', control_port, savefile=True)
hashedPassword = subprocess.Popen([torbinary.tor_binary(),
'--hash-password',
plaintext],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(hashedPassword.stdout.readline, b''):
password = line.decode()
if 'warn' not in password:
break
torrc_data = """SocksPort """ + str(socks_port) + """ OnionTrafficOnly
DataDirectory """ + home_dir + """tordata/
CookieAuthentication 1
KeepalivePeriod 40
CircuitsAvailableTimeout 86400
ControlPort """ + str(control_port) + """
HashedControlPassword """ + str(password) + """
"""
if config.get('general.security_level', 1) == 0:
torrc_data += """\nHiddenServiceDir """ + home_dir + """hs/
\n""" + hs_ver + """\n
HiddenServiceNumIntroductionPoints 20
HiddenServiceMaxStreams 500
HiddenServiceMaxStreamsCloseCircuit 1
HiddenServicePort 80 """ + api_server_ip + """:""" + str(hs_port)
torrc_data = add_bridges(torrc_data)
torrc_data += customtorrc.get_custom_torrc()
torrc = open(tor_config_location, 'w')
torrc.write(torrc_data)
torrc.close()
def __init__(self, hsPort, apiServerIP='127.0.0.1'):
# set data dir
self.dataDir = identifyhome.identify_home()
self.socksPort = getopenport.get_open_port()
self.torConfigLocation = self.dataDir + 'torrc'
self.readyState = False
self.hsPort = hsPort
self._torInstnace = ''
self.myID = ''
self.apiServerIP = apiServerIP
self.torBinary = torbinary.tor_binary()
def get_post_by_board_with_offset(board, offset):
offset = int(offset)
OFFSET_COUNT = 10
board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/board-index.cache.json', flush_on_exit=False)
board_cache.refresh()
posts = board_cache.get(board)
if posts is None:
posts = ''
else:
posts.reverse()
posts = ','.join(posts[offset:offset + OFFSET_COUNT])
return Response(posts)
def test_contact_get_info(self):
contact = crypto.generate()[0]
contact = contactmanager.ContactManager(contact, saveUser=True)
fileLocation = '%s/contacts/%s.json' % (identifyhome.identify_home(),
contact.publicKey)
with open(fileLocation, 'w') as contactFile:
contactFile.write('{"alias": "bob"}')
self.assertEqual(contact.get_info('alias', forceReload=True), 'bob')
self.assertEqual(contact.get_info('fail', forceReload=True), None)
self.assertEqual(contact.get_info('fail'), None)
def test_contact_set_info(self):
contact = crypto.generate()[0]
contact = contactmanager.ContactManager(contact, saveUser=True)
fileLocation = '%s/contacts/%s.json' % (identifyhome.identify_home(),
contact.publicKey)
contact.set_info('alias', 'bob')
self.assertTrue(os.path.exists(fileLocation))
with open(fileLocation, 'r') as data:
data = data.read()
data = json.loads(data)
self.assertEqual(data['alias'], 'bob')
def load_inbox():
inbox_list = []
deleted = simplekv.DeadSimpleKV(identifyhome.identify_home() +
'/mailcache.dat').get('deleted_mail')
if deleted is None:
deleted = []
for blockHash in blockmetadb.get_blocks_by_type('pm'):
block = onionrblockapi.Block(blockHash)
block.decrypt()
if block.decrypted and reconstructhash.deconstruct_hash(
blockHash) not in deleted:
inbox_list.append(blockHash)
return inbox_list
def version(verbosity = 5, function = logger.info):
'''
Displays the Onionr version
'''
function('Onionr v%s (%s) (API v%s)' % (onionrvalues.ONIONR_VERSION, platform.machine(), onionrvalues.API_VERSION), terminal=True)
if verbosity >= 1:
function(onionrvalues.ONIONR_TAGLINE, terminal=True)
if verbosity >= 2:
pf = platform.platform()
release = platform.release()
python_imp = platform.python_implementation()
python_version = platform.python_version()
function(f'{python_imp} {python_version} on {pf} {release}', terminal=True)
function('Onionr data dir: %s' % identifyhome.identify_home(), terminal=True)
def __init__(self, hsPort, apiServerIP='127.0.0.1'):
# set data dir
self.dataDir = identifyhome.identify_home()
self.torConfigLocation = self.dataDir + 'torrc'
self.readyState = False
self.socksPort = getopenport.get_open_port()
self.hsPort = hsPort
self._torInstnace = ''
self.myID = ''
self.apiServerIP = apiServerIP
if os.path.exists('./tor'):
self.torBinary = './tor'
elif os.path.exists('/usr/bin/tor'):
self.torBinary = '/usr/bin/tor'
else:
self.torBinary = 'tor'
def offset_reader_endpoint(name):
if not name[:-4].isalnum():
return Response(400, "Path must be alphanumeric except for file ext")
path = identify_home() + name
if not exists(path):
return Response(404, "Path not found in Onionr data directory")
offset = request.args.get('offset')
if not offset:
offset = 0
else:
offset = int(offset)
result = read_from_offset(path, offset)._asdict()
return ujson.dumps(result, reject_bytes=False)
def show_details():
"""Print out details.
node transport address(es)
active user ID
active user ID in mnemonic form
"""
details = {
'Data directory': identifyhome.identify_home(),
'Node Address': gethostname.get_hostname(),
'Public Key': onionrcrypto.pub_key.replace('=', ''),
'Human-readable Public Key': mnemonickeys.get_human_readable_ID()
}
for detail in details:
logger.info('%s%s: \n%s%s\n' %
(logger.colors.fg.lightgreen, detail,
logger.colors.fg.green, details[detail]),
terminal=True)
def __init__(self, publicKey, saveUser=False, recordExpireSeconds=5):
try:
if mnemonickeys.DELIMITER in publicKey:
publicKey = mnemonickeys.get_base32(publicKey)
#publicKey = unpaddedbase32.b32encode(bytesconverter.str_to_bytes(publicKey))
except ValueError:
pass
publicKey = bytesconverter.bytes_to_str(
unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)))
super(ContactManager, self).__init__(publicKey, saveUser=saveUser)
home = identifyhome.identify_home()
self.dataDir = home + '/contacts/'
self.dataFile = '%s/contacts/%s.json' % (home, publicKey)
self.lastRead = 0
self.recordExpire = recordExpireSeconds
self.data = self._loadData()
self.deleted = False
if not os.path.exists(self.dataDir):
os.mkdir(self.dataDir)
def test_delete_contact(self):
contact = crypto.generate()[0]
contact = contactmanager.ContactManager(contact, saveUser=True)
fileLocation = '%s/contacts/%s.json' % (identifyhome.identify_home(),
contact.publicKey)
self.assertFalse(os.path.exists(fileLocation))
with open(fileLocation, 'w') as contactFile:
contactFile.write('{"alias": "test"}')
self.assertTrue(os.path.exists(fileLocation))
contact.delete_contact()
self.assertFalse(os.path.exists(fileLocation))
try:
contact.get_info('alias')
except onionrexceptions.ContactDeleted:
pass
else:
self.assertTrue(False)
try:
contact.set_info('alias', 'test2')
except onionrexceptions.ContactDeleted:
pass
else:
self.assertTrue(False)
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import os, re, importlib
from . import onionrevents as events
import config, logger
from utils import identifyhome
# set data dir
dataDir = identifyhome.identify_home()
_pluginsfolder = dataDir + 'plugins/'
_instances = dict()
config.reload()
def reload(stop_event=True):
'''
Reloads all the plugins
'''
check()
try:
enabled_plugins = get_enabled_plugins()
def on_softreset(api, data=None):
try:
os.remove(identifyhome.identify_home() + '/board-index.cache.json')
logger.info('Cleared Circles board cache')
except FileNotFoundError:
pass
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import os
from utils import identifyhome
data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home())
# Use the bitwise operators to merge these settings
USE_ANSI = 0b100
if os.name == 'nt':
USE_ANSI = 0b000
OUTPUT_TO_CONSOLE = 0b010
OUTPUT_TO_FILE = 0b001
LEVEL_DEBUG = 1
LEVEL_INFO = 2
LEVEL_WARN = 3
LEVEL_ERROR = 4
LEVEL_FATAL = 5
LEVEL_IMPORTANT = 6
MAX_LOG_FILE_LINES = 10000
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
flask_blueprint = Blueprint('circles', __name__)
root = os.path.dirname(os.path.realpath(__file__))
with open(os.path.dirname(os.path.realpath(__file__)) + '/info.json',
'r') as info_file:
data = info_file.read().strip()
version = json.loads(data)['version']
BOARD_CACHE_FILE = identifyhome.identify_home() + '/board-index.cache.json'
read_only_cache = DeadSimpleKV(BOARD_CACHE_FILE,
flush_on_exit=False,
refresh_seconds=30)
@flask_blueprint.route('/board/<path:path>', endpoint='circlesstatic')
def load_mail(path):
return send_from_directory(root + '/web/', path)
@flask_blueprint.route('/board/', endpoint='circlesindex')
def load_mail_index():
return send_from_directory(root + '/web/', 'index.html')
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sys, os, json
from flask import Response, request, redirect, Blueprint, abort
from onionrusers import contactmanager
from onionrutils import stringvalidators
from utils import reconstructhash, identifyhome
import filepaths
import deadsimplekv as simplekv
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import loadinbox, sentboxdb
flask_blueprint = Blueprint('mail', __name__)
kv = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/mailcache.dat')
@flask_blueprint.route('/mail/ping')
def mail_ping():
return 'pong!'
@flask_blueprint.route('/mail/deletemsg/<block>', methods=['POST'])
def mail_delete(block):
if not stringvalidators.validate_hash(block):
abort(504)
block = reconstructhash.deconstruct_hash(block)
existing = kv.get('deleted_mail')
if existing is None:
existing = []
def daemon():
"""Start Onionr's primary threads for communicator, API server, node, and LAN."""
def _handle_sig_term(signum, frame):
pid = str(os.getpid())
main_pid = localcommand.local_command('/getpid')
#logger.info(main_pid, terminal=True)
if main_pid and main_pid == pid:
logger.info(
f"Received sigterm, shutting down gracefully. PID: {pid}", terminal=True)
localcommand.local_command('/shutdownclean')
else:
logger.info(
f"Recieved sigterm in child process or fork, exiting. PID: {pid}")
sys.exit(0)
signal.signal(signal.SIGTERM, _handle_sig_term)
# Determine if Onionr is in offline mode.
# When offline, Onionr can only use LAN and disk transport
offline_mode = config.get('general.offline_mode', False)
if not hastor.has_tor():
offline_mode = True
logger.error("Tor is not present in system path or Onionr directory",
terminal=True)
# Create shared objects
shared_state = toomanyobjs.TooMany()
# Add DeadSimpleKV for quasi-global variables (ephemeral key-value)
shared_state.get(DeadSimpleKV)
# Initialize the quasi-global variables
setup_kv(shared_state.get(DeadSimpleKV))
shared_state.get(daemoneventsapi.DaemonEventsBP)
Thread(target=shared_state.get(apiservers.ClientAPI).start,
daemon=True, name='client HTTP API').start()
if not offline_mode:
Thread(target=shared_state.get(apiservers.PublicAPI).start,
daemon=True, name='public HTTP API').start()
# Init run time tester
# (ensures Onionr is running right, for testing purposes)
# Run time tests are not normally run
shared_state.get(runtests.OnionrRunTestManager)
# Create singleton
shared_state.get(serializeddata.SerializedData)
shared_state.share_object() # share the parent object to the threads
show_logo()
# since we randomize loopback API server hostname to protect against attacks,
# we have to wait for it to become set
apiHost = ''
if not offline_mode:
apiHost = get_api_host_until_available()
net = NetController(config.get('client.public.port', 59497),
apiServerIP=apiHost)
shared_state.add(net)
shared_state.get(onionrstatistics.tor.TorStats)
security_level = config.get('general.security_level', 1)
use_existing_tor = config.get('tor.use_existing_tor', False)
if not offline_mode:
# we need to setup tor for use
_setup_online_mode(use_existing_tor, net, security_level)
_show_info_messages()
logger.info(
"Onionr daemon is running under " + str(os.getpid()), terminal=True)
events.event('init', threaded=False)
events.event('daemon_start')
if config.get('transports.lan', True):
if not onionrvalues.IS_QUBES:
Thread(target=LANServer(shared_state).start_server,
daemon=True).start()
LANManager(shared_state).start()
else:
logger.warn('LAN not supported on Qubes', terminal=True)
if config.get('transports.sneakernet', True):
Thread(target=sneakernet_import_thread, daemon=True).start()
Thread(target=statistics_reporter,
args=[shared_state], daemon=True).start()
shared_state.get(DeadSimpleKV).put(
'proxyPort', net.socksPort)
spawn_client_threads(shared_state)
clean_blocks_not_meeting_pow(shared_state)
communicator.startCommunicator(shared_state)
clean_ephemeral_services()
if not offline_mode and not use_existing_tor:
net.killTor()
else:
try:
os.remove(filepaths.tor_hs_address_file)
except FileNotFoundError:
pass
better_sleep(5)
cleanup.delete_run_files()
if security_level >= 2:
filenuke.nuke.clean_tree(identifyhome.identify_home())
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
direct_blueprint = Blueprint('chat', __name__)
key_store = simplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=5)
storage_dir = identifyhome.identify_home()
@direct_blueprint.before_request
def request_setup():
key_store.refresh()
host = request.host
host = host.strip('.b32.i2p')
host = host.strip('.onion')
g.host = host
g.peer = key_store.get('dc-' + g.host)
@direct_blueprint.route('/chat/ping')
def pingdirect():
return 'pong!'
from utils import identifyhome
import os
home = identifyhome.identify_home()
if not home.endswith('/'): home += '/'
app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../'
usage_file = home + 'disk-usage.txt'
block_data_location = home + 'blocks/'
contacts_location = home + 'contacts/'
public_API_host_file = home + 'public-host.txt'
private_API_host_file = home + 'private-host.txt'
bootstrap_file_location = 'static-data/bootstrap-nodes.txt'
data_nonce_file = home + 'block-nonces.dat'
forward_keys_file = home + 'forward-keys.db'
cached_storage = home + 'cachedstorage.dat'
announce_cache = home + 'announcecache.dat'
export_location = home + 'block-export/'
upload_list = home + 'upload-list.json'
config_file = home + 'config.json'
daemon_mark_file = home + '/daemon-true.txt'
lock_file = home + 'onionr.lock'
site_cache = home + 'onionr-sites.txt'
tor_hs_loc = home + 'hs/'
tor_hs_address_file = home + 'hs/hostname'
run_check_file = home + '.runcheck'
data_nonce_file = home + 'block-nonces.dat'