def _handle_peers_message(self, node, message):
received = time.time()
message = read_peers(node.server.btctxstore, message)
if message is None:
return # dont care about this message
with self.pipeline_mutex:
data = self.pipeline_scanning.get(message.sender)
if data is None:
return # not being scanned
_log.info("Received peers from {0}!".format(
node_id_to_address(message.sender))
)
data["latency"]["peers"] = received - data["latency"]["peers"]
data["peers"] = storjnode.util.chunks(message.body, 20)
# add previously unknown peers
for peer in data["peers"]:
scanning = peer in self.pipeline_scanning
scanned = peer in self.pipeline_scanned
processed = peer in self.pipeline_processed
testing_bandwith = (self.pipeline_bandwith_test is not None and
peer == self.pipeline_bandwith_test[0])
if not (scanning or scanned or processed or testing_bandwith):
self.pipeline_scanning[peer] = copy.deepcopy(DEFAULT_DATA)
self._check_scan_complete(message.sender, data)
def read(btctxstore, message):
# FIXME make sure body does not contain dicts
if not isinstance(message, list):
return None
if len(message) != 5:
return None
msg = Message(*message)
if not isinstance(msg.sender, bytes):
return None
if len(msg.sender) != 20:
return None
if not isinstance(msg.version, int):
return None
if msg.version < 0:
return None
# token and body must be checked by caller
if not isinstance(msg.rawsig, bytes):
return None
if len(msg.rawsig) != 65:
return None
# verify signature
address = node_id_to_address(msg.sender)
signature = base64.b64encode(msg.rawsig)
data = binascii.hexlify(umsgpack.packb([msg.version, msg.token, msg.body]))
if btctxstore.verify_signature(address, signature, data):
return msg
return None
def _process_scanning(self, nodeid, data):
# request with exponential backoff
now = time.time()
window = storjnode.network.WALK_TIMEOUT ** data["request"]["tries"]
if time.time() < data["request"]["last"] + window:
return # wait for response
_log.info("Requesting info/peers for {0}, try {1}!".format(
node_id_to_address(nodeid),
data["request"]["tries"]
))
# request peers
if data["peers"] is None:
request_peers(self.node, nodeid)
if data["latency"]["peers"] is None:
data["latency"]["peers"] = now
# request info
if data["network"] is None:
request_info(self.node, nodeid)
if data["latency"]["info"] is None:
data["latency"]["info"] = now
data["request"]["last"] = now
data["request"]["tries"] = data["request"]["tries"] + 1
def create_shard(node, num, begin, end, scanned):
# encode scanned data
encoded_scanned = {}
for nodeid, data in scanned:
node_address = node_id_to_address(nodeid)
data["peers"] = [node_id_to_address(p) for p in data["peers"]]
encoded_scanned[node_address] = data
# write info to shard
shard = BytesIO()
shard.write(json.dumps({
"node": node.get_address(),
"num": num,
"begin": begin,
"end": end,
"scanned": encoded_scanned,
}))
return shard
def create_shard(node, num, begin, end, processed):
# encode processed data
encoded_processed = {}
if processed:
for nodeid, data in processed.items():
node_address = node_id_to_address(nodeid)
data["peers"] = [node_id_to_address(p) for p in data["peers"]]
del data["request"]
encoded_processed[node_address] = data
# write info to shard
shard = BytesIO()
shard.write(json.dumps({
"node": node.get_address(),
"num": num,
"begin": begin,
"end": end,
"processed": encoded_processed,
}, indent=2))
return shard
def _check_scan_complete(self, nodeid, data):
if data["peers"] is None:
return # peers not yet received
if data["network"] is None:
return # info not yet received
# move to scanned
del self.scanning[nodeid]
self.scanned[nodeid] = data
txt = "Processed {0}, scanned {1}, scanning {2}!"
_log.info(txt.format(
node_id_to_address(nodeid),
len(self.scanned), len(self.scanning)
))
def _handle_info_message(self, node, source_id, message):
received = time.time()
message = read_info(node.server.btctxstore, message)
if message is None:
return # dont care about this message
with self.mutex:
data = self.scanning.get(message.sender)
if data is None:
return # not being scanned
_log.info("Received info from {0}!".format(
node_id_to_address(message.sender))
)
data["latency"]["info"] = received - data["latency"]["info"]
data["storage"] = message.body.storage
data["network"] = message.body.network
data["version"] = (message.version, message.body.version)
self._check_scan_complete(message.sender, data)
def _handle_peers_message(self, node, source_id, message):
received = time.time()
message = read_peers(node.server.btctxstore, message)
if message is None:
return # dont care about this message
with self.mutex:
data = self.scanning.get(message.sender)
if data is None:
return # not being scanned
_log.info("Received peers from {0}!".format(
node_id_to_address(message.sender))
)
data["latency"]["peers"] = received - data["latency"]["peers"]
data["peers"] = storjnode.util.chunks(message.body, 20)
for peer in data["peers"]:
if (peer not in self.scanned and peer not in self.scanning):
self.scanning[peer] = copy.deepcopy(DEFAULT_DATA)
self._check_scan_complete(message.sender, data)
def _check_scan_complete(self, nodeid, data):
# expect caller to have pipeline mutex
if data["peers"] is None:
return # peers not yet received
if data["network"] is None:
return # info not yet received
# move to scanned
del self.pipeline_scanning[nodeid]
self.pipeline_scanned[nodeid] = data
txt = ("Scan complete for {0}, "
"scanned:{1}, scanning:{2}, processed:{3}!")
_log.info(txt.format(
node_id_to_address(nodeid),
len(self.pipeline_scanned),
len(self.pipeline_scanning),
len(self.pipeline_processed),
))
def _handle_info_message(self, node, message):
received = time.time()
message = read_info(node.server.btctxstore, message)
if message is None:
return # dont care about this message
with self.pipeline_mutex:
data = self.pipeline_scanning.get(message.sender)
if data is None:
return # not being scanned
_log.info("Received info from {0}!".format(
node_id_to_address(message.sender))
)
data["latency"]["info"] = received - data["latency"]["info"]
data["version"] = {
"protocol": message.version,
"storjnode": message.body.version
}
data["storage"] = message.body.storage._asdict()
data["network"] = message.body.network._asdict()
data["platform"] = message.body.platform._asdict()
self._check_scan_complete(message.sender, data)
def verify_signature(msg, wif, node_id=None): # FIXME use read instead
assert(isinstance(msg, OrderedDict))
if u"signature" not in msg:
return 0
msg = msg.copy() # work on a copy for thread saftey
sig = msg.pop("signature")
# Use our address.
api = BtcTxStore(testnet=False, dryrun=True)
try:
if node_id is None:
address = api.get_address(wif)
ret = api.verify_signature_unicode(address, sig, str(msg))
else:
address = node_id_to_address(node_id)
ret = api.verify_signature_unicode(address, sig, str(msg))
except TypeError:
return 0
return ret
def _process_bandwidth_test(self):
# expects caller to have pipeline mutex
not_testing_bandwith = self.pipeline_bandwith_test is None
if (not_testing_bandwith and len(self.pipeline_scanned) > 0):
# pop first entry
nodeid = self.pipeline_scanned.keys()[0]
data = self.pipeline_scanned[nodeid]
del self.pipeline_scanned[nodeid]
# XXX skip bandwith test
self.pipeline_processed[nodeid] = data
return
_log.info("Starting bandwith test for: {0}".format(
node_id_to_address(nodeid))
)
# start bandwith test (timeout after 5min)
self.pipeline_bandwith_test = (nodeid, data)
deferred = self.node.test_bandwidth(nodeid)
deferred.addCallback(self._handle_bandwith_test_success)
deferred.addErrback(self._handle_bandwith_test_error)
def _handle_bandwith_test_success(self, results):
with self.pipeline_mutex:
assert(results[0])
nodeid, data = self.pipeline_bandwith_test
# save test results
data["bandwidth"] = {
"send": results[1]["upload"],
"receive": results[1]["download"]
}
# move peer to processed
self.pipeline_processed[nodeid] = data
txt = "Processed:{0}, scanned:{1}, scanning:{2}, processed:{3}!"
_log.info(txt.format(
node_id_to_address(nodeid),
len(self.pipeline_scanned),
len(self.pipeline_scanning),
len(self.pipeline_processed),
))
# free up bandwith test for next peer
self.pipeline_bandwith_test = None
