def setUpClass(cls):
cls.alice_wallet = BtcTxStore(testnet=False, dryrun=True)
cls.alice_wif = cls.alice_wallet.create_key()
cls.alice_node_id = address_to_node_id(cls.alice_wallet.get_address(cls.alice_wif))
cls.alice_dht = pyp2p.dht_msg.DHT(node_id=cls.alice_node_id, networking=0)
cls.alice_transfer = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=0,
debug=1,
wan_ip="8.8.8.8",
dht_node=cls.alice_dht,
),
BandwidthLimit(),
wif=cls.alice_wif,
store_config={tempfile.mkdtemp(): None},
)
cls.alice_test = BandwidthTest(cls.alice_wif, cls.alice_transfer, cls.alice_dht, 0)
# Bob sample node.
cls.bob_wallet = BtcTxStore(testnet=False, dryrun=True)
cls.bob_wif = cls.bob_wallet.create_key()
cls.bob_node_id = address_to_node_id(cls.bob_wallet.get_address(cls.bob_wif))
cls.bob_dht = pyp2p.dht_msg.DHT(node_id=cls.bob_node_id, networking=0)
cls.bob_transfer = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=0,
debug=1,
wan_ip="8.8.8.8",
dht_node=cls.bob_dht,
),
BandwidthLimit(),
wif=cls.bob_wif,
store_config={tempfile.mkdtemp(): None},
)
cls.bob_test = BandwidthTest(cls.bob_wif, cls.bob_transfer, cls.bob_dht, 0)
def alice_hook_relay_message(node_id, req):
_log.debug(str(req))
cls.req = req
def bob_hook_relay_message(node_id, req):
_log.debug(str(req))
cls.alice_test.api.relay_message = alice_hook_relay_message
cls.bob_test.api.relay_message = bob_hook_relay_message
cls.alice_test.start(cls.bob_transfer.net.unl.value)
def setUp(self):
self.test_storage_dir = tempfile.mkdtemp()
# Sample node.
self.wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True)
self.wif = self.wallet.get_key(self.wallet.create_wallet())
self.node_id = address_to_node_id(self.wallet.get_address(self.wif))
self.store_config = {
os.path.join(self.test_storage_dir, "storage"): {"limit": 0}
}
self.dht_node = pyp2p.dht_msg.DHT(
node_id=self.node_id,
networking=0
)
# Transfer client.
self.client = FileTransfer(
pyp2p.net.Net(
node_type="simultaneous",
nat_type="preserving",
net_type="direct",
passive_port=60500,
dht_node=self.dht_node,
wan_ip="8.8.8.8",
debug=1
),
wif=self.wif,
store_config=self.store_config
)
def _setup_data_transfer_client(self, store_config, passive_port,
passive_bind, node_type, nat_type, wan_ip):
# Setup handlers for callbacks registered via the API.
handlers = {
"complete": self._transfer_complete_handlers,
"accept": self._transfer_request_handlers
}
wallet = BtcTxStore(testnet=False, dryrun=True)
wif = self.get_key()
node_id = address_to_node_id(wallet.get_address(wif))
#dht_node = SimDHT(node_id=node_id)
dht_node = self
self._data_transfer = FileTransfer(
net=Net(
net_type="direct",
node_type=node_type,
nat_type=nat_type,
dht_node=dht_node,
debug=1,
passive_port=passive_port,
passive_bind=passive_bind,
wan_ip=wan_ip
),
wif=wif,
store_config=store_config,
handlers=handlers
)
# Setup success callback values.
self._data_transfer.success_value = (self.sync_get_wan_ip(), self.port)
self.process_data_transfers()
def process_unl_requests(node, src_id, msg):
unl = node._data_transfer.net.unl.value
try:
msg = OrderedDict(msg)
# Not a UNL request.
if msg[u"type"] != u"unl_request":
return
# Check signature.
their_node_id = address_to_node_id(msg[u"requester"])
if not verify_signature(msg, node.get_key(), their_node_id):
return
# Response.
response = sign(OrderedDict(
{
u"type": u"unl_response",
u"requestee": node.get_address(),
u"unl": unl
}
), node.get_key())
# Send response.
node.relay_message(their_node_id, response.items())
except (ValueError, KeyError) as e:
global _log
_log.debug(str(e))
_log.debug("Protocol: invalid JSON")
def setUp(self):
self.test_storage_dir = tempfile.mkdtemp()
# Sample node.
self.wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True)
self.wif = self.wallet.get_key(self.wallet.create_wallet())
self.node_id = address_to_node_id(self.wallet.get_address(self.wif))
self.store_config = {
os.path.join(self.test_storage_dir, "storage"): {"limit": 0}
}
# dht_node = pyp2p.dht_msg.DHT(node_id=node_id)
self.dht_node = storjnode.network.Node(
self.wif, bootstrap_nodes=DEFAULT_BOOTSTRAP_NODES,
disable_data_transfer=True
)
# Transfer client.
self.client = FileTransfer(
pyp2p.net.Net(
node_type="simultaneous",
nat_type="preserving",
net_type="direct",
passive_port=0,
dht_node=self.dht_node,
debug=1
),
BandwidthLimit(),
wif=self.wif,
store_config=self.store_config
)
# Accept all transfers.
def accept_handler(contract_id, src_unl, data_id, file_size):
return 1
# Add accept handler.
self.client.handlers["accept"].add(accept_handler)
# Accept UNL requests.
enable_unl_requests(self.dht_node)
def create(btctxstore, node_wif, token, body):
# FIXME make sure body does not contain dicts
# sign message data (version + token + body)
data = binascii.hexlify(umsgpack.packb([PROTOCOL_VERSION, token, body]))
signature = btctxstore.sign_data(node_wif, data)
# use compact unencoded data to conserve package bytes
nodeid = address_to_node_id(btctxstore.get_address(node_wif))
rawsig = base64.b64decode(signature)
message = Message(nodeid, PROTOCOL_VERSION, token, body, rawsig)
# check if message to large
packed_message = umsgpack.packb(message)
if len(packed_message) > MAX_MESSAGE_DATA:
txt = "Message size {0} > {1} allowed."
raise MaxSizeExceeded(
txt.format(len(packed_message), MAX_MESSAGE_DATA)
)
return message
def process_unl_requests(node, msg):
_log.debug("In process unl requests: ")
_log.debug(msg)
unl = node._data_transfer.net.unl.value
_log.debug(unl)
try:
msg = list_to_ordered_dict(msg)
# Not a UNL request.
if msg[u"type"] != u"unl_request":
_log.debug("unl req: invalid type")
_log.debug(msg[u"type"])
return
# Check signature.
their_node_id = address_to_node_id(msg[u"requester"])
if not verify_signature(msg, node.get_key(), their_node_id):
_log.debug("unl req: invalid sig")
return
# Response.
response = sign(OrderedDict(
{
u"type": u"unl_response",
u"requestee": node.get_address(),
u"unl": unl
}
), node.get_key())
# Send response.
response = ordered_dict_to_list(response)
node.repeat_relay_message(their_node_id, response)
except (ValueError, KeyError):
_log.debug("val err or key err")
pass # not a unl request
def test_queued():
from crochet import setup
setup()
# Alice sample node.
alice_wallet = BtcTxStore(testnet=False, dryrun=True)
alice_wif = alice_wallet.create_key()
alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif))
alice_dht = pyp2p.dht_msg.DHT(
node_id=alice_node_id,
networking=0
)
alice = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63400,
dht_node=alice_dht,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=alice_wif,
store_config={tempfile.mkdtemp(): None},
)
# Bob sample node.
bob_wallet = BtcTxStore(testnet=False, dryrun=True)
bob_wif = bob_wallet.create_key()
bob_node_id = address_to_node_id(bob_wallet.get_address(bob_wif))
bob_dht = pyp2p.dht_msg.DHT(
node_id=bob_node_id,
networking=0
)
bob = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63401,
dht_node=bob_dht,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=bob_wif,
store_config={tempfile.mkdtemp(): None}
)
# Simulate Alice + Bob "connecting"
alice_dht.add_relay_link(bob_dht)
bob_dht.add_relay_link(alice_dht)
# Accept all transfers.
def accept_handler(contract_id, src_unl, data_id, file_size):
return 1
# Add accept handler.
alice.handlers["accept"].add(accept_handler)
bob.handlers["accept"].add(accept_handler)
# Create file we're suppose to be uploading.
data_id = ("5feceb66ffc86f38d952786c6d696c"
"79c2dbc239dd4e91b46729d73a27fb57e9")
path = os.path.join(list(alice.store_config)[0], data_id)
if not os.path.exists(path):
with open(path, "w") as fp:
fp.write("0")
# Alice wants to upload data to Bob.
upload_contract_id = alice.data_request(
"download",
data_id,
0,
bob.net.unl.value
)
# Delete source file.
def callback_builder(path, alice, bob, data_id):
def callback(client, contract_id, con):
print("Upload succeeded")
print("Removing content and downloading back")
os.remove(path)
# Fix transfers.
bob.handlers["complete"] = []
# Synchronize cons and check con.unl.
time.sleep(1)
clients = {"alice": alice, "bob": bob}
for client in list({"alice": alice, "bob": bob}):
print()
print(client)
clients[client].net.synchronize()
nodes_out = clients[client].net.outbound
nodes_in = clients[client].net.inbound
for node in nodes_out + nodes_in:
print(node["con"].unl)
print(clients[client].cons)
# Queued transfer:
download_contract_id = alice.data_request(
"upload",
data_id,
0,
bob.net.unl.value
)
print("Download contract ID =")
print(download_contract_id)
# Indicate Bob's download succeeded.
def alice_callback(val):
print("Download succeeded")
global queue_succeeded
queue_succeeded = 1
def alice_errback(val):
print("Download failed! Error:")
print(val)
# Hook upload from bob.
d = alice.defers[download_contract_id]
d.addCallback(alice_callback)
d.addErrback(alice_errback)
return callback
# Register callback for bob (when he's downloaded the data.)
bob.handlers["complete"] = [
callback_builder(path, alice, bob, data_id)
]
# d = alice.defers[upload_contract_id]
# d.addCallback(callback_builder(path, alice, bob, data_id))
# Main event loop.
timeout = time.time() + 40
while not queue_succeeded and time.time() < timeout:
for client in [alice, bob]:
if client == alice:
_log.debug("Alice")
else:
_log.debug("Bob")
process_transfers(client)
time.sleep(1)
if not queue_succeeded:
print("\a")
for client in [alice, bob]:
client.net.stop()
assert(queue_succeeded == 1)
# Find temp file path.
path = self.downloading[data_id]
_log.debug(path)
with open(path, "ab") as fp:
fp.write(chunk)
if __name__ == "__main__":
from crochet import setup
setup()
# Alice sample node.
alice_wallet = BtcTxStore(testnet=False, dryrun=True)
alice_wif = alice_wallet.create_key()
alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif))
# print(type(alice_node_id))
alice_dht_node = pyp2p.dht_msg.DHT(node_id=alice_node_id)
# print(alice_dht_node.get_id())
alice_dht_node = storjnode.network.Node(
alice_wif, bootstrap_nodes=[("240.0.0.0", 1337)],
disable_data_transfer=True
)
alice = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
from threading import Thread
print("")
print("------- P2P File Sharing ------")
print("")
print("")
print("Starting DHT and Networking ... Please wait")
print("")
test_storage_dir = tempfile.mkdtemp()
# Sample node.
wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True)
wif = wallet.get_key(wallet.create_wallet())
node_id = address_to_node_id(wallet.get_address(wif))
store_config = {
os.path.join(test_storage_dir, "storage"): {"limit": 0}
}
dht_node = storjnode.network.Node(
wif, bootstrap_nodes=DEFAULT_BOOTSTRAP_NODES,
disable_data_transfer=True
)
# Transfer client.
client = FileTransfer(
pyp2p.net.Net(
net_type="direct",
passive_port=0,
dht_node=dht_node,
def test_bandwidth_test(self):
# Alice sample node.
alice_wallet = BtcTxStore(testnet=False, dryrun=True)
alice_wif = alice_wallet.create_key()
alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif))
alice_dht = pyp2p.dht_msg.DHT(
node_id=alice_node_id,
networking=0
)
alice_transfer = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63600,
debug=1,
wan_ip="8.8.8.8",
dht_node=alice_dht,
),
wif=alice_wif,
store_config={tempfile.mkdtemp(): None}
)
_log.debug("Alice UNL")
_log.debug(alice_transfer.net.unl.value)
# Bob sample node.
bob_wallet = BtcTxStore(testnet=False, dryrun=True)
bob_wif = bob_wallet.create_key()
bob_node_id = address_to_node_id(bob_wallet.get_address(bob_wif))
bob_dht = pyp2p.dht_msg.DHT(
node_id=bob_node_id,
networking=0
)
bob_transfer = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63601,
debug=1,
wan_ip="8.8.8.8",
dht_node=bob_dht
),
wif=bob_wif,
store_config={tempfile.mkdtemp(): None}
)
# Link DHT nodes.
alice_dht.add_relay_link(bob_dht)
bob_dht.add_relay_link(alice_dht)
_log.debug("Bob UNL")
_log.debug(bob_transfer.net.unl.value)
# Show bandwidth.
def show_bandwidth(results):
global test_success
test_success = 1
_log.debug(results)
# Test bandwidth between Alice and Bob.
bob_test = BandwidthTest(bob_wif, bob_transfer, bob_dht, 0)
alice_test = BandwidthTest(alice_wif, alice_transfer, alice_dht, 0)
d = alice_test.start(bob_transfer.net.unl.value)
d.addCallback(show_bandwidth)
# Main event loop.
# and not test_success
end_time = time.time() + 60
while alice_test.active_test is not None and time.time() < end_time:
for client in [alice_transfer, bob_transfer]:
process_transfers(client)
time.sleep(0.002)
# End net.
for client in [alice_transfer, bob_transfer]:
client.net.stop()
self.assertTrue(test_success == 1)
def get_id(self):
if self._cached_id is not None:
return self._cached_id
address = self._btctxstore.get_address(self.key)
self._cached_id = util.address_to_node_id(address)
return self._cached_id
def setUp(self):
# Alice
self.alice_wallet = BtcTxStore(testnet=False, dryrun=True)
self.alice_wif = "L18vBLrz3A5QxJ6K4bUraQQZm6BAdjuAxU83e16y3x7eiiHTApHj"
self.alice_node_id = address_to_node_id(
self.alice_wallet.get_address(self.alice_wif)
)
self.alice_dht_node = pyp2p.dht_msg.DHT(
node_id=self.alice_node_id,
networking=0
)
self.alice_storage = tempfile.mkdtemp()
self.alice = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=0,
dht_node=self.alice_dht_node,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=self.alice_wif,
store_config={self.alice_storage: None}
)
# Bob
self.bob_wallet = BtcTxStore(testnet=False, dryrun=True)
self.bob_wif = "L3DBWWbuL3da2x7qAmVwBpiYKjhorJuAGobecCYQMCV7tZMAnDsr"
self.bob_node_id = address_to_node_id(
self.bob_wallet.get_address(self.bob_wif))
self.bob_dht_node = pyp2p.dht_msg.DHT(
node_id=self.bob_node_id,
networking=0
)
self.bob_storage = tempfile.mkdtemp()
self.bob = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=0,
dht_node=self.bob_dht_node,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=self.bob_wif,
store_config={self.bob_storage: None}
)
# Accept all transfers.
def accept_handler(contract_id, src_unl, data_id, file_size):
return 1
# Add accept handler.
self.alice.handlers["accept"].add(accept_handler)
self.bob.handlers["accept"].add(accept_handler)
# Link DHT nodes.
self.alice_dht_node.add_relay_link(self.bob_dht_node)
self.bob_dht_node.add_relay_link(self.alice_dht_node)
# Bypass sending messages for client.
def send_msg(dict_obj, unl):
print("Skipped sending message in test")
print(dict_obj)
print(unl)
# Install send msg hooks.
self.alice.send_msg = send_msg
self.bob.send_msg = send_msg
# Bypass sending relay messages for clients.
def relay_msg(node_id, msg):
print("Skipping relay message in test")
print(node_id)
print(msg)
# Install relay msg hooks.
if self.alice.net.dht_node is not None:
self.alice.net.dht_node.relay_message = relay_msg
if self.bob.net.dht_node is not None:
self.bob.net.dht_node.relay_message = relay_msg
# Bypass UNL.connect for clients.
def unl_connect(their_unl, events, force_master=1, hairpin=1,
nonce="0" * 64):
print("Skipping UNL.connect!")
print("Their unl = ")
print(their_unl)
print("Events = ")
print(events)
print("Force master = ")
print(force_master)
print("Hairpin = ")
print(hairpin)
print("Nonce = ")
print(nonce)
# Install UNL connect hooks.
self.alice.net.unl.connect = unl_connect
self.bob.net.unl.connect = unl_connect
# Record syn.
data_id = u"5feceb66ffc86f38d952786c6d696c79"
data_id += u"c2dbc239dd4e91b46729d73a27fb57e9"
self.syn = OrderedDict([
(u"status", u"SYN"),
(u"data_id", data_id),
(u"file_size", 100),
(u"host_unl", self.alice.net.unl.value),
(u"dest_unl", self.bob.net.unl.value),
(u"src_unl", self.alice.net.unl.value)
])
def test_multiple_transfers(self):
def make_random_file(file_size=1024 * 100,
directory=self.test_storage_dir):
content = os.urandom(file_size)
file_name = hashlib.sha256(content[0:64]).hexdigest()
path = storjnode.util.full_path(os.path.join(directory, file_name))
with open(path, "wb") as fp:
fp.write(content)
return {"path": path, "content": content}
# Sample node.
wallet = btctxstore.BtcTxStore(testnet=False, dryrun=True)
wif = wallet.get_key(wallet.create_wallet())
node_id = address_to_node_id(wallet.get_address(wif))
store_config = {
os.path.join(self.test_storage_dir, "storage"): {
"limit": 0
}
}
#dht_node = pyp2p.dht_msg.DHT(node_id=node_id)
dht_node = storjnode.network.Node(
wif,
bootstrap_nodes=DEFAULT_BOOTSTRAP_NODES,
disable_data_transfer=True)
client = FileTransfer(pyp2p.net.Net(node_type="simultaneous",
nat_type="preserving",
net_type="direct",
passive_port=60400,
dht_node=dht_node,
debug=1),
wif=wif,
store_config=store_config)
#print("Giving nodes some time to find peers.")
time.sleep(storjnode.network.WALK_TIMEOUT)
dht_node.refresh_neighbours()
time.sleep(storjnode.network.WALK_TIMEOUT)
_log.debug("Net started")
# Make random file
rand_file_infos = [make_random_file()]
# Move file to storage directory.
file_infos = [client.move_file_to_storage(rand_file_infos[0]["path"])]
# Delete original file.
os.remove(rand_file_infos[0]["path"])
_log.debug("Testing upload")
# Upload file from storage.
for file_info in file_infos:
client.data_request("upload", file_info["data_id"], 0,
TEST_NODE["unl"])
# Process file transfers.
duration = 15
timeout = time.time() + duration
while time.time() <= timeout or client.is_queued():
process_transfers(client)
time.sleep(0.002)
# Check upload exists.
for i in range(0, 1):
url = TEST_NODE["web"] + file_infos[i]["data_id"]
r = requests.get(url, timeout=3)
if r.status_code != 200:
_log.debug(r.status_code)
assert (0)
else:
assert (r.content == rand_file_infos[i]["content"])
_log.debug("File upload succeeded.")
# Delete storage file copy.
client.remove_file_from_storage(file_infos[0]["data_id"])
# Download file from storage.
_log.debug("Testing download.")
for file_info in file_infos:
client.data_request("download", file_info["data_id"], 0,
TEST_NODE["unl"])
# Process file transfers.
duration = 15
timeout = time.time() + duration
while time.time() <= timeout or client.is_queued():
process_transfers(client)
time.sleep(0.002)
# Check we received this file.
for i in range(0, 1):
path = storage.manager.find(store_config, file_infos[i]["data_id"])
if not os.path.isfile(path):
assert (0)
else:
with open(path, "r") as fp:
content = fp.read()
assert (content == rand_file_infos[i]["content"])
# Delete storage file copy.
client.remove_file_from_storage(file_infos[0]["data_id"])
# Stop networking.
dht_node.stop()
_log.debug("Download succeeded.")
