def test_inserted_housekeeping(testmanager):
"""Tests that inserted blocks are proprely deleted"""
bl = insert('testdata', expire=12)
wait_seconds = 132 # Wait two minutes plus expire time
count = 0
if bl in get_block_list():
while count < wait_seconds:
if bl in get_block_list():
sleep(0.8)
count += 1
else:
return
raise ValueError('Inserted block with expiry not erased')
else:
raise ValueError('Inserted block in expiry test not present in list')
def get_stats(self):
"""Return statistics about our node"""
stats = {}
proc = Process()
def get_open_files():
if WINDOWS: return proc.num_handles()
return proc.num_fds()
try:
self._too_many
except AttributeError:
sleep(1)
comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon,
args=(self._too_many, ))
connected = []
[
connected.append(x) for x in comm_inst.onlinePeers
if x not in connected
]
stats['uptime'] = comm_inst.getUptime()
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(comm_inst.blockQueue)
stats['threads'] = proc.num_threads()
stats['ramPercent'] = proc.memory_percent()
stats['fd'] = get_open_files()
stats['diskUsage'] = human_size(size(identify_home()))
return json.dumps(stats)
def import_new_blocks(scanDir=''):
'''
This function is intended to scan for new blocks ON THE DISK and import them
'''
blockList = blockmetadb.get_block_list()
exist = False
if scanDir == '':
scanDir = filepaths.block_data_location
if not scanDir.endswith('/'):
scanDir += '/'
for block in glob.glob(scanDir + "*%s" % (BLOCK_EXPORT_FILE_EXT, )):
if block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT,
'') not in blockList:
exist = True
logger.info('Found new block on dist %s' % block, terminal=True)
with open(block, 'rb') as newBlock:
block = block.replace(scanDir,
'').replace(BLOCK_EXPORT_FILE_EXT, '')
if crypto.hashers.sha3_hash(newBlock.read()) == block.replace(
BLOCK_EXPORT_FILE_EXT, ''):
blockmetadb.add_to_block_DB(block.replace(
BLOCK_EXPORT_FILE_EXT, ''),
dataSaved=True)
logger.info('Imported block %s' % block, terminal=True)
blockmetadata.process_block_metadata(block)
else:
logger.warn('Failed to verify hash for %s' % block,
terminal=True)
if not exist:
logger.info('No blocks found to import', terminal=True)
def get_stats(self):
"""Return statistics about our node"""
stats = {}
proc = Process()
def get_open_files():
if WINDOWS:
return proc.num_handles()
return proc.num_fds()
try:
self._too_many
except AttributeError:
sleep(1)
kv: "DeadSimpleKV" = self._too_many.get_by_string("DeadSimpleKV")
connected = []
[
connected.append(x) for x in kv.get('onlinePeers')
if x not in connected
]
stats['uptime'] = get_epoch() - kv.get('startTime')
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(kv.get('blockQueue'))
stats['threads'] = proc.num_threads()
stats['ramPercent'] = proc.memory_percent()
stats['fd'] = get_open_files()
stats['diskUsage'] = human_size(size(identify_home()))
return json.dumps(stats)
def clean_blocks_not_meeting_pow(shared_state):
"""Clean blocks not meeting min send/rec pow. Used if config.json POW changes"""
block_list = blockmetadb.get_block_list()
for block in block_list:
if not hashMeetsDifficulty(block):
logger.warn(
f"Deleting block {block} because it was stored" +
"with a POW level smaller than current.", terminal=True)
__purge_block(shared_state, block)
def delete_plaintext_no_blacklist():
"""Delete, but do not blacklist, plaintext blocks."""
block_list = blockmetadb.get_block_list()
for block in block_list:
block = Block(hash=block)
if not block.isEncrypted:
remove_block(block.hash)
onionrstorage.deleteBlock(block.hash)
def get_public_block_list(publicAPI, request):
# Provide a list of our blocks, with a date offset
dateAdjust = request.args.get('date')
bList = blockmetadb.get_block_list(dateRec=dateAdjust)
share_list = ''
if config.get('general.hide_created_blocks', True):
for b in publicAPI.hideBlocks:
if b in bList:
# Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
bList.remove(b)
for b in bList:
share_list += '%s\n' % (reconstructhash.deconstruct_hash(b), )
return Response(share_list)
def getData(name):
resp = ""
if stringvalidators.validate_hash(name):
if name in blockmetadb.get_block_list():
try:
resp = client_get_block.get_block_data(name, decrypt=True)
except ValueError:
pass
else:
abort(404)
else:
abort(404)
return Response(resp)
def test_sneakernet_import(test_manager):
in_db = lambda b: b in get_block_list()
bl = insert(os.urandom(10))
assert in_db(bl)
export_block(bl)
assert os.path.exists(export_location + bl + BLOCK_EXPORT_FILE_EXT)
remove_block(bl)
deleteBlock(bl)
assert not in_db(bl)
os.remove(data_nonce_file)
move(export_location + bl + BLOCK_EXPORT_FILE_EXT, block_data_location)
sleep(1)
assert in_db(bl)
def _sync_peer(url):
our_blocks = get_block_list()
blocks = requests.get(url + 'blist/0').text.splitlines()
for block in blocks:
if block not in our_blocks and hashMeetsDifficulty(block):
try:
import_block_from_data(
requests.get(
url + f'get/{block}', stream=True).raw.read(6000000))
except onionrexceptions.InvalidMetadata:
logger.warn(f"Could not get {block} from lan peer")
except onionrexceptions.InvalidProof:
logger.warn(
f"Invalid proof for {block} from lan peer {peer}", terminal=True)
break
def clean_old_blocks(shared_state):
"""Delete expired blocks + old blocks if disk allocation is near full"""
blacklist = onionrblacklist.OnionrBlackList()
# Delete expired blocks
for bHash in blockmetadb.expiredblocks.get_expired_blocks():
__purge_block(shared_state, bHash, add_to_blacklist=True)
logger.info('Deleted expired block: %s' % (bHash,))
while storage_counter.is_full():
try:
oldest = blockmetadb.get_block_list()[0]
except IndexError:
break
else:
__purge_block(shared_state, bHash, add_to_blacklist=True)
logger.info('Deleted block because of full storage: %s' % (oldest,))
def test_export(self):
testargs = ["onionr.py", "circlesend", "tests", "hello"]
with patch.object(sys, 'argv', testargs):
try:
parser.register()
except SystemExit:
pass
bl = blockmetadb.get_block_list()[0]
testargs = ["onionr.py", "export-block", bl]
with patch.object(sys, 'argv', testargs):
parser.register()
with open(export_location + '/' + bl + BLOCK_EXPORT_FILE_EXT,
'rb') as f:
if b'hello' not in f.read():
raise ValueError('No exported block')
def get_public_block_list(public_API, request):
# Provide a list of our blocks, with a date offset
date_adjust = request.args.get('date')
type_filter = request.args.get('type')
b_list = blockmetadb.get_block_list(date_rec=date_adjust)
share_list = ''
if config.get('general.hide_created_blocks', True):
for b in public_API.hideBlocks:
if b in b_list:
# Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
b_list.remove(b)
for b in b_list:
if type_filter:
if Block(b, decrypt=False).getType() != type_filter:
continue
share_list += '%s\n' % (reconstructhash.deconstruct_hash(b), )
return Response(share_list)
def should_download(comm_inst, block_hash) -> bool:
"""Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
should = True
if block_hash in blockmetadb.get_block_list():
# Don't download block we have
should = False
else:
if blacklist.inBlacklist(block_hash):
# Don't download blacklisted block
should = False
if should is False:
# Remove block from communicator queue if it shouldn't be downloaded
try:
del comm_inst.blockQueue[block_hash]
except KeyError:
pass
return should
def should_download(shared_state, block_hash) -> bool:
"""Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
should = True
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if block_hash in blockmetadb.get_block_list():
# Don't download block we have
should = False
else:
if blacklist.inBlacklist(block_hash):
# Don't download blacklisted block
should = False
if should is False:
# Remove block from communicator queue if it shouldn't be downloaded
try:
del kv.get('blockQueue')[block_hash]
except KeyError:
pass
return should
def get_stats(self):
"""Return statistics about our node"""
stats = {}
try:
self._too_many
except AttributeError:
time.sleep(1)
comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon,
args=(self._too_many, ))
connected = []
[
connected.append(x) for x in comm_inst.onlinePeers
if x not in connected
]
stats['uptime'] = comm_inst.getUptime()
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(comm_inst.blockQueue)
return json.dumps(stats)
def get_block_data(publicAPI, data):
'''data is the block hash in hex'''
resp = ''
if stringvalidators.validate_hash(data):
if not config.get('general.hide_created_blocks',
True) or data not in publicAPI.hideBlocks:
if data in blockmetadb.get_block_list():
block = apiutils.GetBlockData().get_block_data(data, raw=True)
try:
block = block.encode() # Encode in case data is binary
except AttributeError:
if len(block) == 0:
abort(404)
block = bytesconverter.str_to_bytes(block)
resp = block
if len(resp) == 0:
abort(404)
resp = ""
# Has to be octet stream, otherwise binary data fails hash check
return Response(resp, mimetype='application/octet-stream')
def clean_old_blocks(comm_inst):
'''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
blacklist = onionrblacklist.OnionrBlackList()
# Delete expired blocks
for bHash in blockmetadb.expiredblocks.get_expired_blocks():
blacklist.addToDB(bHash)
removeblock.remove_block(bHash)
onionrstorage.deleteBlock(bHash)
__remove_from_upload(comm_inst, bHash)
logger.info('Deleted block: %s' % (bHash, ))
while comm_inst.storage_counter.is_full():
oldest = blockmetadb.get_block_list()[0]
blacklist.addToDB(oldest)
removeblock.remove_block(oldest)
onionrstorage.deleteBlock(oldest)
__remove_from_upload.remove(comm_inst, oldest)
logger.info('Deleted block: %s' % (oldest, ))
comm_inst.decrementThreadCount('clean_old_blocks')
def clean_old_blocks(comm_inst):
"""Delete expired blocks + old blocks if disk allocation is near full"""
blacklist = onionrblacklist.OnionrBlackList()
# Delete expired blocks
for bHash in blockmetadb.expiredblocks.get_expired_blocks():
blacklist.addToDB(bHash)
removeblock.remove_block(bHash)
onionrstorage.deleteBlock(bHash)
__remove_from_upload(comm_inst, bHash)
logger.info('Deleted block: %s' % (bHash, ))
while storage_counter.is_full():
try:
oldest = blockmetadb.get_block_list()[0]
except IndexError:
break
else:
blacklist.addToDB(oldest)
removeblock.remove_block(oldest)
onionrstorage.deleteBlock(oldest)
__remove_from_upload(comm_inst, oldest)
logger.info('Deleted block: %s' % (oldest, ))
comm_inst.decrementThreadCount('clean_old_blocks')
import sys
import os
import stem
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
from coredb.blockmetadb import get_block_list
from onionrblocks.onionrblockapi import Block
for bl in get_block_list():
bl_obj = Block(bl, decrypt=False)
b_type = bl_obj.getType()
if not b_type:
b_type = "encrypted"
print(bl + " - " + str(bl_obj.date) + " - " + b_type)
def lookup_blocks_from_communicator(comm_inst):
logger.info('Looking up new blocks')
tryAmount = 2
newBlocks = ''
existingBlocks = blockmetadb.get_block_list() # List of existing saved blocks
triedPeers = [] # list of peers we've tried this time around
maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
lastLookupTime = 0 # Last time we looked up a particular peer's list
new_block_count = 0
for i in range(tryAmount):
listLookupCommand = 'getblocklist' # This is defined here to reset it each time
if len(comm_inst.blockQueue) >= maxBacklog:
break
if not comm_inst.isOnline:
break
# check if disk allocation is used
if comm_inst.storage_counter.is_full():
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
break
peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer
# if we've already tried all the online peers this time around, stop
if peer in triedPeers:
if len(comm_inst.onlinePeers) == len(triedPeers):
break
else:
continue
triedPeers.append(peer)
# Get the last time we looked up a peer's stamp to only fetch blocks since then.
# Saved in memory only for privacy reasons
try:
lastLookupTime = comm_inst.dbTimestamps[peer]
except KeyError:
lastLookupTime = 0
else:
listLookupCommand += '?date=%s' % (lastLookupTime,)
try:
newBlocks = peeraction.peer_action(comm_inst, peer, listLookupCommand) # get list of new block hashes
except Exception as error:
logger.warn('Could not get new blocks from %s.' % peer, error = error)
newBlocks = False
else:
comm_inst.dbTimestamps[peer] = epoch.get_rounded_epoch(roundS=60)
if newBlocks != False:
# if request was a success
for i in newBlocks.split('\n'):
if stringvalidators.validate_hash(i):
i = reconstructhash.reconstruct_hash(i)
# if newline seperated string is valid hash
if not i in existingBlocks:
# if block does not exist on disk and is not already in block queue
if i not in comm_inst.blockQueue:
if onionrproofs.hashMeetsDifficulty(i) and not blacklist.inBlacklist(i):
if len(comm_inst.blockQueue) <= 1000000:
comm_inst.blockQueue[i] = [peer] # add blocks to download queue
new_block_count += 1
else:
if peer not in comm_inst.blockQueue[i]:
if len(comm_inst.blockQueue[i]) < 10:
comm_inst.blockQueue[i].append(peer)
if new_block_count > 0:
block_string = ""
if new_block_count > 1:
block_string = "s"
logger.info('Discovered %s new block%s' % (new_block_count, block_string), terminal=True)
comm_inst.download_blocks_timer.count = int(comm_inst.download_blocks_timer.frequency * 0.99)
comm_inst.decrementThreadCount('lookup_blocks_from_communicator')
return
def _sync_peer(url):
our_blocks = get_block_list()
blocks = requests.get(url + 'blist/0').text.splitlines()
for block in blocks:
if block not in our_blocks:
import_block_from_data(requests.get(url + f'get/{block}', stream=True).raw.read(6000000))
def refresh_db(self):
self.check_time = get_epoch()
if not self.block_type:
self.block_list = get_block_list()
else:
self.block_list = get_blocks_by_type(self.block_type)
def show_stats():
"""Print/log statistic info about our Onionr install."""
try:
# define stats messages here
totalBlocks = len(blockmetadb.get_block_list())
home = identifyhome.identify_home()
totalBanned = len(onionrblacklist.OnionrBlackList().getList())
messages = {
# info about local client
'Onionr Daemon Status':
((logger.colors.fg.green + 'Online') if check_communicator(
timeout=9) else logger.colors.fg.red + 'Offline'),
# file and folder size stats
'div1':
True, # this creates a solid line across the screen, a div
'Total Block Size':
sizeutils.human_size(sizeutils.size(home + 'blocks/')),
'Total Plugin Size':
sizeutils.human_size(sizeutils.size(home + 'plugins/')),
'Log File Size':
sizeutils.human_size(sizeutils.size(home + 'output.log')),
# count stats
'div2':
True,
'Known Peers (nodes)':
str(max(len(keydb.listkeys.list_adders()) - 1, 0)),
'Enabled Plugins':
str(len(config.get('plugins.enabled', list()))) + ' / ' +
str(len(os.listdir(home + 'plugins/'))),
'Stored Blocks':
str(totalBlocks),
'Deleted Blocks':
str(totalBanned)
}
# color configuration
colors = {
'title': logger.colors.bold,
'key': logger.colors.fg.lightgreen,
'val': logger.colors.fg.green,
'border': logger.colors.fg.lightblue,
'reset': logger.colors.reset
}
# pre-processing
maxlength = 0
width = getconsolewidth.get_console_width()
for key, val in messages.items():
if not (type(val) is bool and val is True):
maxlength = max(len(key), maxlength)
prewidth = maxlength + len(' | ')
groupsize = width - prewidth - len('[+] ')
# generate stats table
logger.info(colors['title'] +
'Onionr v%s Statistics' % onionrvalues.ONIONR_VERSION +
colors['reset'],
terminal=True)
logger.info(colors['border'] + '-' * (maxlength + 1) + '+' +
colors['reset'],
terminal=True)
for key, val in messages.items():
if not (type(val) is bool and val is True):
val = [
str(val)[i:i + groupsize]
for i in range(0, len(str(val)), groupsize)
]
logger.info(colors['key'] + str(key).rjust(maxlength) +
colors['reset'] + colors['border'] + ' | ' +
colors['reset'] + colors['val'] + str(val.pop(0)) +
colors['reset'],
terminal=True)
for value in val:
logger.info(' ' * maxlength + colors['border'] + ' | ' +
colors['reset'] + colors['val'] + str(value) +
colors['reset'],
terminal=True)
else:
logger.info(colors['border'] + '-' * (maxlength + 1) + '+' +
colors['reset'],
terminal=True)
logger.info(colors['border'] + '-' * (maxlength + 1) + '+' +
colors['reset'],
terminal=True)
except Exception as e: # pylint: disable=W0703
logger.error('Failed to generate statistics table. ' + str(e),
error=e,
timestamp=False,
terminal=True)
def lookup_blocks_from_communicator(comm_inst):
logger.info('Looking up new blocks')
tryAmount = 2
newBlocks = ''
# List of existing saved blocks
existingBlocks = get_block_list()
triedPeers = [] # list of peers we've tried this time around
# Max amount of *new* block hashes to have in queue
maxBacklog = 1560
lastLookupTime = 0 # Last time we looked up a particular peer's list
new_block_count = 0
for i in range(tryAmount):
# Defined here to reset it each time, time offset is added later
listLookupCommand = 'getblocklist'
if len(comm_inst.blockQueue) >= maxBacklog:
break
if not comm_inst.isOnline:
break
# check if disk allocation is used
if comm_inst.storage_counter.is_full():
logger.debug(
'Not looking up new blocks due to maximum amount of disk used')
break
try:
# select random online peer
peer = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
time.sleep(1)
continue
# if we've already tried all the online peers this time around, stop
if peer in triedPeers:
if len(comm_inst.onlinePeers) == len(triedPeers):
break
else:
continue
triedPeers.append(peer)
# Get the last time we looked up a peer's stamp,
# to only fetch blocks since then.
# Saved in memory only for privacy reasons
try:
lastLookupTime = comm_inst.dbTimestamps[peer]
except KeyError:
lastLookupTime = epoch.get_epoch() - \
config.get("general.max_block_age",
onionrvalues.DEFAULT_EXPIRE)
listLookupCommand += '?date=%s' % (lastLookupTime,)
try:
newBlocks = peeraction.peer_action(
comm_inst,
peer, listLookupCommand) # get list of new block hashes
except Exception as error:
logger.warn(
f'Could not get new blocks from {peer}.',
error=error)
newBlocks = False
if newBlocks != False: # noqa
# if request was a success
for i in newBlocks.split('\n'):
if stringvalidators.validate_hash(i):
i = reconstructhash.reconstruct_hash(i)
# if newline seperated string is valid hash
# if block does not exist on disk + is not already in queue
if i not in existingBlocks:
if i not in comm_inst.blockQueue:
if onionrproofs.hashMeetsDifficulty(i) and \
not blacklist.inBlacklist(i):
if len(comm_inst.blockQueue) <= 1000000:
# add blocks to download queue
comm_inst.blockQueue[i] = [peer]
new_block_count += 1
comm_inst.dbTimestamps[peer] = \
epoch.get_rounded_epoch(roundS=60)
else:
if peer not in comm_inst.blockQueue[i]:
if len(comm_inst.blockQueue[i]) < 10:
comm_inst.blockQueue[i].append(peer)
if new_block_count > 0:
block_string = ""
if new_block_count > 1:
block_string = "s"
logger.info(
f'Discovered {new_block_count} new block{block_string}',
terminal=True)
comm_inst.download_blocks_timer.count = \
int(comm_inst.download_blocks_timer.frequency * 0.99)
comm_inst.decrementThreadCount('lookup_blocks_from_communicator')
def get_block_list_for_lan(time):
return Response('\n'.join(get_block_list(date_rec=time)))
