def upload_blocks_from_communicator(comm_inst):
# when inserting a block, we try to upload it to a few peers to add some deniability
triedPeers = []
finishedUploads = []
comm_inst.blocksToUpload = comm_inst._core._crypto.randomShuffle(
comm_inst.blocksToUpload)
if len(comm_inst.blocksToUpload) != 0:
for bl in comm_inst.blocksToUpload:
if not comm_inst._core._utils.validateHash(bl):
logger.warn('Requested to upload invalid block')
comm_inst.decrementThreadCount('uploadBlock')
return
for i in range(min(len(comm_inst.onlinePeers), 6)):
peer = comm_inst.pickOnlinePeer()
if peer in triedPeers:
continue
triedPeers.append(peer)
url = 'http://' + peer + '/upload'
data = {'block': block.Block(bl).getRaw()}
proxyType = proxypicker.pick_proxy(peer)
logger.info("Uploading block to " + peer)
if not comm_inst._core._utils.doPostRequest(
url, data=data, proxyType=proxyType) == False:
comm_inst._core._utils.localCommand('waitforshare/' + bl,
post=True)
finishedUploads.append(bl)
for x in finishedUploads:
try:
comm_inst.blocksToUpload.remove(x)
except ValueError:
pass
comm_inst.decrementThreadCount('uploadBlock')
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
TIMER_NAME = "upload_blocks_from_communicator"
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = comm_inst.shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
kv.put('blocksToUpload',
onionrcrypto.cryptoutils.random_shuffle(kv.get('blocksToUpload')))
def remove_from_hidden(bl):
sleep(60)
try:
comm_inst.shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(bl)
except ValueError:
pass
if len(kv.get('blocksToUpload')) != 0:
for bl in kv.get('blocksToUpload'):
if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True)
comm_inst.decrementThreadCount(TIMER_NAME)
return
session = session_manager.add_session(bl)
for _ in range(min(len(kv.get('onlinePeers')), 6)):
try:
peer = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
continue
try:
session.peer_exists[peer]
continue
except KeyError:
pass
try:
if session.peer_fails[peer] > 3:
continue
except KeyError:
pass
if peer in tried_peers:
continue
tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
except onionrexceptions.NoDataAvailable:
finishedUploads.append(bl)
break
proxy_type = proxypicker.pick_proxy(peer)
logger.info(f"Uploading block {bl[:8]} to {peer}",
terminal=True)
resp = basicrequests.do_post_request(
url,
data=data,
proxyType=proxy_type,
content_type='application/octet-stream')
if resp is not False:
if resp == 'success':
Thread(target=remove_from_hidden,
args=[bl],
daemon=True).start()
session.success()
session.peer_exists[peer] = True
elif resp == 'exists':
session.success()
session.peer_exists[peer] = True
else:
session.fail()
session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp}',
terminal=True)
else:
session.fail()
session_manager.clean_session()
for x in finishedUploads:
try:
kv.get('blocksToUpload').remove(x)
comm_inst.shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(x)
except ValueError:
pass
comm_inst.decrementThreadCount(TIMER_NAME)
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
"""Accepts a communicator instance and uploads blocks from its upload queue"""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
TIMER_NAME = "upload_blocks_from_communicator"
session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(
sessionmanager.BlockUploadSessionManager)
triedPeers = []
finishedUploads = []
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(
comm_inst.blocksToUpload)
if len(comm_inst.blocksToUpload) != 0:
for bl in comm_inst.blocksToUpload:
if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True)
comm_inst.decrementThreadCount(TIMER_NAME)
return
session = session_manager.add_session(bl)
for i in range(min(len(comm_inst.onlinePeers), 6)):
peer = onlinepeers.pick_online_peer(comm_inst)
try:
session.peer_exists[peer]
continue
except KeyError:
pass
try:
if session.peer_fails[peer] > 3: continue
except KeyError:
pass
if peer in triedPeers: continue
triedPeers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
except onionrexceptions.NoDataAvailable:
finishedUploads.append(bl)
break
proxyType = proxypicker.pick_proxy(peer)
logger.info(f"Uploading block {bl[:8]} to {peer}",
terminal=True)
resp = basicrequests.do_post_request(
url,
data=data,
proxyType=proxyType,
content_type='application/octet-stream')
if not resp == False:
if resp == 'success':
session.success()
session.peer_exists[peer] = True
elif resp == 'exists':
session.success()
session.peer_exists[peer] = True
else:
session.fail()
session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp[:15]}',
terminal=True)
else:
session.fail()
session_manager.clean_session()
for x in finishedUploads:
try:
comm_inst.blocksToUpload.remove(x)
except ValueError:
pass
comm_inst.decrementThreadCount(TIMER_NAME)
def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
SystemRandom().shuffle(kv.get('blocksToUpload'))
def remove_from_hidden(bl):
sleep(60)
try:
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(bl)
except ValueError:
pass
if len(kv.get('blocksToUpload')) != 0:
for bl in kv.get('blocksToUpload'):
if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True)
return
session = session_manager.add_session(bl)
for _ in range(min(len(kv.get('onlinePeers')), 6)):
try:
peer = onlinepeers.pick_online_peer(kv)
if not block.Block(bl).isEncrypted:
if peer in kv.get('plaintextDisabledPeers'):
logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa
continue
except onionrexceptions.OnlinePeerNeeded:
continue
try:
session.peer_exists[peer]
continue
except KeyError:
pass
try:
if session.peer_fails[peer] > 3:
continue
except KeyError:
pass
if peer in tried_peers:
continue
tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
if not data:
logger.warn(
f"Couldn't get data for block in upload list {bl}",
terminal=True)
raise onionrexceptions.NoDataAvailable
try:
def __check_metadata():
metadata = get_block_metadata_from_data(data)[0]
if not validate_metadata(metadata, data):
logger.warn(
f"Metadata for uploading block not valid {bl}")
raise onionrexceptions.InvalidMetadata
__check_metadata()
except onionrexceptions.DataExists:
pass
except( # noqa
onionrexceptions.NoDataAvailable,
onionrexceptions.InvalidMetadata) as _:
finishedUploads.append(bl)
break
proxy_type = proxypicker.pick_proxy(peer)
logger.info(
f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(
url, data=data, proxyType=proxy_type,
content_type='application/octet-stream')
if resp is not False:
if resp == 'success':
Thread(target=remove_from_hidden,
args=[bl], daemon=True).start()
session.success()
session.peer_exists[peer] = True
elif resp == 'exists':
session.success()
session.peer_exists[peer] = True
else:
session.fail()
session.fail_peer(peer)
shared_state.get_by_string(
'OnionrCommunicatorDaemon').getPeerProfileInstance(
peer).addScore(-5)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp}',
terminal=True)
else:
session.fail()
session_manager.clean_session()
for x in finishedUploads:
try:
kv.get('blocksToUpload').remove(x)
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(x)
except ValueError:
pass