def setUpClass(cls): cls.alice_wallet = BtcTxStore(testnet=False, dryrun=True) cls.alice_wif = cls.alice_wallet.create_key() cls.alice_node_id = address_to_node_id(cls.alice_wallet.get_address(cls.alice_wif)) cls.alice_dht = pyp2p.dht_msg.DHT(node_id=cls.alice_node_id, networking=0) cls.alice_transfer = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=0, debug=1, wan_ip="8.8.8.8", dht_node=cls.alice_dht, ), BandwidthLimit(), wif=cls.alice_wif, store_config={tempfile.mkdtemp(): None}, ) cls.alice_test = BandwidthTest(cls.alice_wif, cls.alice_transfer, cls.alice_dht, 0) # Bob sample node. cls.bob_wallet = BtcTxStore(testnet=False, dryrun=True) cls.bob_wif = cls.bob_wallet.create_key() cls.bob_node_id = address_to_node_id(cls.bob_wallet.get_address(cls.bob_wif)) cls.bob_dht = pyp2p.dht_msg.DHT(node_id=cls.bob_node_id, networking=0) cls.bob_transfer = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=0, debug=1, wan_ip="8.8.8.8", dht_node=cls.bob_dht, ), BandwidthLimit(), wif=cls.bob_wif, store_config={tempfile.mkdtemp(): None}, ) cls.bob_test = BandwidthTest(cls.bob_wif, cls.bob_transfer, cls.bob_dht, 0) def alice_hook_relay_message(node_id, req): _log.debug(str(req)) cls.req = req def bob_hook_relay_message(node_id, req): _log.debug(str(req)) cls.alice_test.api.relay_message = alice_hook_relay_message cls.bob_test.api.relay_message = bob_hook_relay_message cls.alice_test.start(cls.bob_transfer.net.unl.value)
def setUp(self): self.test_storage_dir = tempfile.mkdtemp() # Sample node. self.wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True) self.wif = self.wallet.get_key(self.wallet.create_wallet()) self.node_id = address_to_node_id(self.wallet.get_address(self.wif)) self.store_config = { os.path.join(self.test_storage_dir, "storage"): {"limit": 0} } self.dht_node = pyp2p.dht_msg.DHT( node_id=self.node_id, networking=0 ) # Transfer client. self.client = FileTransfer( pyp2p.net.Net( node_type="simultaneous", nat_type="preserving", net_type="direct", passive_port=60500, dht_node=self.dht_node, wan_ip="8.8.8.8", debug=1 ), wif=self.wif, store_config=self.store_config )
def _setup_data_transfer_client(self, store_config, passive_port, passive_bind, node_type, nat_type, wan_ip): # Setup handlers for callbacks registered via the API. handlers = { "complete": self._transfer_complete_handlers, "accept": self._transfer_request_handlers } wallet = BtcTxStore(testnet=False, dryrun=True) wif = self.get_key() node_id = address_to_node_id(wallet.get_address(wif)) #dht_node = SimDHT(node_id=node_id) dht_node = self self._data_transfer = FileTransfer( net=Net( net_type="direct", node_type=node_type, nat_type=nat_type, dht_node=dht_node, debug=1, passive_port=passive_port, passive_bind=passive_bind, wan_ip=wan_ip ), wif=wif, store_config=store_config, handlers=handlers ) # Setup success callback values. self._data_transfer.success_value = (self.sync_get_wan_ip(), self.port) self.process_data_transfers()
def process_unl_requests(node, src_id, msg): unl = node._data_transfer.net.unl.value try: msg = OrderedDict(msg) # Not a UNL request. if msg[u"type"] != u"unl_request": return # Check signature. their_node_id = address_to_node_id(msg[u"requester"]) if not verify_signature(msg, node.get_key(), their_node_id): return # Response. response = sign(OrderedDict( { u"type": u"unl_response", u"requestee": node.get_address(), u"unl": unl } ), node.get_key()) # Send response. node.relay_message(their_node_id, response.items()) except (ValueError, KeyError) as e: global _log _log.debug(str(e)) _log.debug("Protocol: invalid JSON")
def setUp(self): self.test_storage_dir = tempfile.mkdtemp() # Sample node. self.wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True) self.wif = self.wallet.get_key(self.wallet.create_wallet()) self.node_id = address_to_node_id(self.wallet.get_address(self.wif)) self.store_config = { os.path.join(self.test_storage_dir, "storage"): {"limit": 0} } # dht_node = pyp2p.dht_msg.DHT(node_id=node_id) self.dht_node = storjnode.network.Node( self.wif, bootstrap_nodes=DEFAULT_BOOTSTRAP_NODES, disable_data_transfer=True ) # Transfer client. self.client = FileTransfer( pyp2p.net.Net( node_type="simultaneous", nat_type="preserving", net_type="direct", passive_port=0, dht_node=self.dht_node, debug=1 ), BandwidthLimit(), wif=self.wif, store_config=self.store_config ) # Accept all transfers. def accept_handler(contract_id, src_unl, data_id, file_size): return 1 # Add accept handler. self.client.handlers["accept"].add(accept_handler) # Accept UNL requests. enable_unl_requests(self.dht_node)
def create(btctxstore, node_wif, token, body): # FIXME make sure body does not contain dicts # sign message data (version + token + body) data = binascii.hexlify(umsgpack.packb([PROTOCOL_VERSION, token, body])) signature = btctxstore.sign_data(node_wif, data) # use compact unencoded data to conserve package bytes nodeid = address_to_node_id(btctxstore.get_address(node_wif)) rawsig = base64.b64decode(signature) message = Message(nodeid, PROTOCOL_VERSION, token, body, rawsig) # check if message to large packed_message = umsgpack.packb(message) if len(packed_message) > MAX_MESSAGE_DATA: txt = "Message size {0} > {1} allowed." raise MaxSizeExceeded( txt.format(len(packed_message), MAX_MESSAGE_DATA) ) return message
def process_unl_requests(node, msg): _log.debug("In process unl requests: ") _log.debug(msg) unl = node._data_transfer.net.unl.value _log.debug(unl) try: msg = list_to_ordered_dict(msg) # Not a UNL request. if msg[u"type"] != u"unl_request": _log.debug("unl req: invalid type") _log.debug(msg[u"type"]) return # Check signature. their_node_id = address_to_node_id(msg[u"requester"]) if not verify_signature(msg, node.get_key(), their_node_id): _log.debug("unl req: invalid sig") return # Response. response = sign(OrderedDict( { u"type": u"unl_response", u"requestee": node.get_address(), u"unl": unl } ), node.get_key()) # Send response. response = ordered_dict_to_list(response) node.repeat_relay_message(their_node_id, response) except (ValueError, KeyError): _log.debug("val err or key err") pass # not a unl request
def test_queued(): from crochet import setup setup() # Alice sample node. alice_wallet = BtcTxStore(testnet=False, dryrun=True) alice_wif = alice_wallet.create_key() alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif)) alice_dht = pyp2p.dht_msg.DHT( node_id=alice_node_id, networking=0 ) alice = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=63400, dht_node=alice_dht, wan_ip="8.8.8.8", debug=1 ), BandwidthLimit(), wif=alice_wif, store_config={tempfile.mkdtemp(): None}, ) # Bob sample node. bob_wallet = BtcTxStore(testnet=False, dryrun=True) bob_wif = bob_wallet.create_key() bob_node_id = address_to_node_id(bob_wallet.get_address(bob_wif)) bob_dht = pyp2p.dht_msg.DHT( node_id=bob_node_id, networking=0 ) bob = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=63401, dht_node=bob_dht, wan_ip="8.8.8.8", debug=1 ), BandwidthLimit(), wif=bob_wif, store_config={tempfile.mkdtemp(): None} ) # Simulate Alice + Bob "connecting" alice_dht.add_relay_link(bob_dht) bob_dht.add_relay_link(alice_dht) # Accept all transfers. def accept_handler(contract_id, src_unl, data_id, file_size): return 1 # Add accept handler. alice.handlers["accept"].add(accept_handler) bob.handlers["accept"].add(accept_handler) # Create file we're suppose to be uploading. data_id = ("5feceb66ffc86f38d952786c6d696c" "79c2dbc239dd4e91b46729d73a27fb57e9") path = os.path.join(list(alice.store_config)[0], data_id) if not os.path.exists(path): with open(path, "w") as fp: fp.write("0") # Alice wants to upload data to Bob. upload_contract_id = alice.data_request( "download", data_id, 0, bob.net.unl.value ) # Delete source file. def callback_builder(path, alice, bob, data_id): def callback(client, contract_id, con): print("Upload succeeded") print("Removing content and downloading back") os.remove(path) # Fix transfers. bob.handlers["complete"] = [] # Synchronize cons and check con.unl. time.sleep(1) clients = {"alice": alice, "bob": bob} for client in list({"alice": alice, "bob": bob}): print() print(client) clients[client].net.synchronize() nodes_out = clients[client].net.outbound nodes_in = clients[client].net.inbound for node in nodes_out + nodes_in: print(node["con"].unl) print(clients[client].cons) # Queued transfer: download_contract_id = alice.data_request( "upload", data_id, 0, bob.net.unl.value ) print("Download contract ID =") print(download_contract_id) # Indicate Bob's download succeeded. def alice_callback(val): print("Download succeeded") global queue_succeeded queue_succeeded = 1 def alice_errback(val): print("Download failed! Error:") print(val) # Hook upload from bob. d = alice.defers[download_contract_id] d.addCallback(alice_callback) d.addErrback(alice_errback) return callback # Register callback for bob (when he's downloaded the data.) bob.handlers["complete"] = [ callback_builder(path, alice, bob, data_id) ] # d = alice.defers[upload_contract_id] # d.addCallback(callback_builder(path, alice, bob, data_id)) # Main event loop. timeout = time.time() + 40 while not queue_succeeded and time.time() < timeout: for client in [alice, bob]: if client == alice: _log.debug("Alice") else: _log.debug("Bob") process_transfers(client) time.sleep(1) if not queue_succeeded: print("\a") for client in [alice, bob]: client.net.stop() assert(queue_succeeded == 1)
# Find temp file path. path = self.downloading[data_id] _log.debug(path) with open(path, "ab") as fp: fp.write(chunk) if __name__ == "__main__": from crochet import setup setup() # Alice sample node. alice_wallet = BtcTxStore(testnet=False, dryrun=True) alice_wif = alice_wallet.create_key() alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif)) # print(type(alice_node_id)) alice_dht_node = pyp2p.dht_msg.DHT(node_id=alice_node_id) # print(alice_dht_node.get_id()) alice_dht_node = storjnode.network.Node( alice_wif, bootstrap_nodes=[("240.0.0.0", 1337)], disable_data_transfer=True ) alice = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive",
from threading import Thread print("") print("------- P2P File Sharing ------") print("") print("") print("Starting DHT and Networking ... Please wait") print("") test_storage_dir = tempfile.mkdtemp() # Sample node. wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True) wif = wallet.get_key(wallet.create_wallet()) node_id = address_to_node_id(wallet.get_address(wif)) store_config = { os.path.join(test_storage_dir, "storage"): {"limit": 0} } dht_node = storjnode.network.Node( wif, bootstrap_nodes=DEFAULT_BOOTSTRAP_NODES, disable_data_transfer=True ) # Transfer client. client = FileTransfer( pyp2p.net.Net( net_type="direct", passive_port=0, dht_node=dht_node,
def test_bandwidth_test(self): # Alice sample node. alice_wallet = BtcTxStore(testnet=False, dryrun=True) alice_wif = alice_wallet.create_key() alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif)) alice_dht = pyp2p.dht_msg.DHT( node_id=alice_node_id, networking=0 ) alice_transfer = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=63600, debug=1, wan_ip="8.8.8.8", dht_node=alice_dht, ), wif=alice_wif, store_config={tempfile.mkdtemp(): None} ) _log.debug("Alice UNL") _log.debug(alice_transfer.net.unl.value) # Bob sample node. bob_wallet = BtcTxStore(testnet=False, dryrun=True) bob_wif = bob_wallet.create_key() bob_node_id = address_to_node_id(bob_wallet.get_address(bob_wif)) bob_dht = pyp2p.dht_msg.DHT( node_id=bob_node_id, networking=0 ) bob_transfer = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=63601, debug=1, wan_ip="8.8.8.8", dht_node=bob_dht ), wif=bob_wif, store_config={tempfile.mkdtemp(): None} ) # Link DHT nodes. alice_dht.add_relay_link(bob_dht) bob_dht.add_relay_link(alice_dht) _log.debug("Bob UNL") _log.debug(bob_transfer.net.unl.value) # Show bandwidth. def show_bandwidth(results): global test_success test_success = 1 _log.debug(results) # Test bandwidth between Alice and Bob. bob_test = BandwidthTest(bob_wif, bob_transfer, bob_dht, 0) alice_test = BandwidthTest(alice_wif, alice_transfer, alice_dht, 0) d = alice_test.start(bob_transfer.net.unl.value) d.addCallback(show_bandwidth) # Main event loop. # and not test_success end_time = time.time() + 60 while alice_test.active_test is not None and time.time() < end_time: for client in [alice_transfer, bob_transfer]: process_transfers(client) time.sleep(0.002) # End net. for client in [alice_transfer, bob_transfer]: client.net.stop() self.assertTrue(test_success == 1)
def get_id(self): if self._cached_id is not None: return self._cached_id address = self._btctxstore.get_address(self.key) self._cached_id = util.address_to_node_id(address) return self._cached_id
def setUp(self): # Alice self.alice_wallet = BtcTxStore(testnet=False, dryrun=True) self.alice_wif = "L18vBLrz3A5QxJ6K4bUraQQZm6BAdjuAxU83e16y3x7eiiHTApHj" self.alice_node_id = address_to_node_id( self.alice_wallet.get_address(self.alice_wif) ) self.alice_dht_node = pyp2p.dht_msg.DHT( node_id=self.alice_node_id, networking=0 ) self.alice_storage = tempfile.mkdtemp() self.alice = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=0, dht_node=self.alice_dht_node, wan_ip="8.8.8.8", debug=1 ), BandwidthLimit(), wif=self.alice_wif, store_config={self.alice_storage: None} ) # Bob self.bob_wallet = BtcTxStore(testnet=False, dryrun=True) self.bob_wif = "L3DBWWbuL3da2x7qAmVwBpiYKjhorJuAGobecCYQMCV7tZMAnDsr" self.bob_node_id = address_to_node_id( self.bob_wallet.get_address(self.bob_wif)) self.bob_dht_node = pyp2p.dht_msg.DHT( node_id=self.bob_node_id, networking=0 ) self.bob_storage = tempfile.mkdtemp() self.bob = FileTransfer( pyp2p.net.Net( net_type="direct", node_type="passive", nat_type="preserving", passive_port=0, dht_node=self.bob_dht_node, wan_ip="8.8.8.8", debug=1 ), BandwidthLimit(), wif=self.bob_wif, store_config={self.bob_storage: None} ) # Accept all transfers. def accept_handler(contract_id, src_unl, data_id, file_size): return 1 # Add accept handler. self.alice.handlers["accept"].add(accept_handler) self.bob.handlers["accept"].add(accept_handler) # Link DHT nodes. self.alice_dht_node.add_relay_link(self.bob_dht_node) self.bob_dht_node.add_relay_link(self.alice_dht_node) # Bypass sending messages for client. def send_msg(dict_obj, unl): print("Skipped sending message in test") print(dict_obj) print(unl) # Install send msg hooks. self.alice.send_msg = send_msg self.bob.send_msg = send_msg # Bypass sending relay messages for clients. def relay_msg(node_id, msg): print("Skipping relay message in test") print(node_id) print(msg) # Install relay msg hooks. if self.alice.net.dht_node is not None: self.alice.net.dht_node.relay_message = relay_msg if self.bob.net.dht_node is not None: self.bob.net.dht_node.relay_message = relay_msg # Bypass UNL.connect for clients. def unl_connect(their_unl, events, force_master=1, hairpin=1, nonce="0" * 64): print("Skipping UNL.connect!") print("Their unl = ") print(their_unl) print("Events = ") print(events) print("Force master = ") print(force_master) print("Hairpin = ") print(hairpin) print("Nonce = ") print(nonce) # Install UNL connect hooks. self.alice.net.unl.connect = unl_connect self.bob.net.unl.connect = unl_connect # Record syn. data_id = u"5feceb66ffc86f38d952786c6d696c79" data_id += u"c2dbc239dd4e91b46729d73a27fb57e9" self.syn = OrderedDict([ (u"status", u"SYN"), (u"data_id", data_id), (u"file_size", 100), (u"host_unl", self.alice.net.unl.value), (u"dest_unl", self.bob.net.unl.value), (u"src_unl", self.alice.net.unl.value) ])
def test_multiple_transfers(self): def make_random_file(file_size=1024 * 100, directory=self.test_storage_dir): content = os.urandom(file_size) file_name = hashlib.sha256(content[0:64]).hexdigest() path = storjnode.util.full_path(os.path.join(directory, file_name)) with open(path, "wb") as fp: fp.write(content) return {"path": path, "content": content} # Sample node. wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True) wif = wallet.get_key(wallet.create_wallet()) node_id = address_to_node_id(wallet.get_address(wif)) store_config = { os.path.join(self.test_storage_dir, "storage"): { "limit": 0 } } #dht_node = pyp2p.dht_msg.DHT(node_id=node_id) dht_node = storjnode.network.Node( wif, bootstrap_nodes=DEFAULT_BOOTSTRAP_NODES, disable_data_transfer=True) client = FileTransfer(pyp2p.net.Net(node_type="simultaneous", nat_type="preserving", net_type="direct", passive_port=60400, dht_node=dht_node, debug=1), wif=wif, store_config=store_config) #print("Giving nodes some time to find peers.") time.sleep(storjnode.network.WALK_TIMEOUT) dht_node.refresh_neighbours() time.sleep(storjnode.network.WALK_TIMEOUT) _log.debug("Net started") # Make random file rand_file_infos = [make_random_file()] # Move file to storage directory. file_infos = [client.move_file_to_storage(rand_file_infos[0]["path"])] # Delete original file. os.remove(rand_file_infos[0]["path"]) _log.debug("Testing upload") # Upload file from storage. for file_info in file_infos: client.data_request("upload", file_info["data_id"], 0, TEST_NODE["unl"]) # Process file transfers. duration = 15 timeout = time.time() + duration while time.time() <= timeout or client.is_queued(): process_transfers(client) time.sleep(0.002) # Check upload exists. for i in range(0, 1): url = TEST_NODE["web"] + file_infos[i]["data_id"] r = requests.get(url, timeout=3) if r.status_code != 200: _log.debug(r.status_code) assert (0) else: assert (r.content == rand_file_infos[i]["content"]) _log.debug("File upload succeeded.") # Delete storage file copy. client.remove_file_from_storage(file_infos[0]["data_id"]) # Download file from storage. _log.debug("Testing download.") for file_info in file_infos: client.data_request("download", file_info["data_id"], 0, TEST_NODE["unl"]) # Process file transfers. duration = 15 timeout = time.time() + duration while time.time() <= timeout or client.is_queued(): process_transfers(client) time.sleep(0.002) # Check we received this file. for i in range(0, 1): path = storage.manager.find(store_config, file_infos[i]["data_id"]) if not os.path.isfile(path): assert (0) else: with open(path, "r") as fp: content = fp.read() assert (content == rand_file_infos[i]["content"]) # Delete storage file copy. client.remove_file_from_storage(file_infos[0]["data_id"]) # Stop networking. dht_node.stop() _log.debug("Download succeeded.")