class TestGetAddress(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_standard(self):
wif = self.api.create_key()
address = self.api.get_address(wif)
self.assertTrue(
validate.is_address_valid(address, allowable_netcodes=['XTN']))
def test_input_validation(self):
# test correct types
a = self.api.get_address(S_WIF)
b = self.api.get_address(B_WIF)
c = self.api.get_address(U_WIF)
self.assertEqual(a, b, c)
# TODO invalid types
# TODO invalid input data
def test_standards_compliant(self):
wif = self.api.create_key()
address = self.api.get_address(S_WIF)
self.assertEqual(address, EXPECTED)
class TestGetAddress(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_standard(self):
wif = self.api.create_key()
address = self.api.get_address(wif)
self.assertTrue(validate.is_address_valid(address, allowable_netcodes=['XTN']))
def test_input_validation(self):
# test correct types
a = self.api.get_address(S_WIF)
b = self.api.get_address(B_WIF)
c = self.api.get_address(U_WIF)
self.assertEqual(a, b, c)
# TODO invalid types
# TODO invalid input data
def test_standards_compliant(self):
wif = self.api.create_key()
address = self.api.get_address(S_WIF)
self.assertEqual(address, EXPECTED)
class AppAuthenticationHeadersTest(unittest.TestCase):
def setUp(self):
app.config["SKIP_AUTHENTICATION"] = False # monkey patch
self.app = app.test_client()
self.btctxstore = BtcTxStore()
db.create_all()
def tearDown(self):
db.session.remove()
db.drop_all()
def test_success(self):
# create header date and authorization signature
wif = self.btctxstore.create_key()
btc_addr = self.btctxstore.get_address(wif)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True, usegmt=True)
message = app.config["ADDRESS"] + " " + header_date
header_authorization = self.btctxstore.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
url = '/api/register/{0}'.format(btc_addr)
rv = self.app.get(url, headers=headers)
data = json.loads(rv.data.decode("utf-8"))
self.assertEqual(btc_addr, data["btc_addr"])
self.assertEqual(rv.status_code, 200)
def test_fail(self):
# register without auth headres fails
btc_addr = self.btctxstore.get_address(self.btctxstore.get_key(self.btctxstore.create_wallet()))
rv = self.app.get('/api/register/{0}'.format(btc_addr))
self.assertEqual(rv.status_code, 401)
# register first because ping is lazy
wif = self.btctxstore.get_key(self.btctxstore.create_wallet())
btc_addr = self.btctxstore.get_address(wif)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True, usegmt=True)
message = app.config["ADDRESS"] + " " + header_date
header_authorization = self.btctxstore.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
url = '/api/register/{0}'.format(btc_addr)
rv = self.app.get(url, headers=headers)
self.assertEqual(rv.status_code, 200)
# ping without auth headres fails
time.sleep(app.config["MAX_PING"])
rv = self.app.get('/api/ping/{0}'.format(btc_addr))
self.assertEqual(rv.status_code, 401)
# set height without auth headres fails
btc_addr = self.btctxstore.get_address(self.btctxstore.get_key(self.btctxstore.create_wallet()))
rv = self.app.get('/api/height/{0}/10'.format(btc_addr))
self.assertEqual(rv.status_code, 401)
def test_fail(self):
# register without auth headres fails
rv = self.app.get('/api/register/{0}'.format(addresses["eta"]))
self.assertEqual(rv.status_code, 401)
# register first because ping is lazy
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True, usegmt=True)
message = app.config["ADDRESS"] + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
url = '/api/register/{0}'.format(address)
rv = self.app.get(url, headers=headers)
self.assertEqual(rv.status_code, 200)
# ping without auth headres fails
time.sleep(app.config["MAX_PING"])
rv = self.app.get('/api/ping/{0}'.format(address))
self.assertEqual(rv.status_code, 401)
# set height without auth headres fails
rv = self.app.get('/api/height/{0}/10'.format(addresses["eta"]))
self.assertEqual(rv.status_code, 401)
def test_core_audit(self):
"""
Test of providing correct arguments to the ``requests.post()``
and returning gotten response object.
"""
test_url_address = 'http://test.url.com'
file_hash = sha256(b'some test data').hexdigest()
seed = sha256(b'some test challenge seed').hexdigest()
btctx_api = BtcTxStore(testnet=True, dryrun=True)
sender_key = btctx_api.create_key()
audit_call_result = core.audit(test_url_address, sender_key,
btctx_api, file_hash, seed)
expected_calls = [call(
urljoin(test_url_address, '/api/audit/'),
data={
'data_hash': file_hash,
'challenge_seed': seed,
},
headers={
'sender-address': btctx_api.get_address(sender_key),
'signature': btctx_api.sign_unicode(sender_key, file_hash),
}
)]
self.assertListEqual(
self.mock_post.call_args_list,
expected_calls,
'In the audit() function requests.post() calls are unexpected'
)
self.assertIs(
self.mock_post.return_value,
audit_call_result,
'Returned value must be the object returned by the '
'``requests.post()``'
)
def test_authenticate_headers_provide(self):
"""
Test of preparing and providing credential headers when ``sender_key``
and ``btctx_api`` are provided.
"""
btctx_api = BtcTxStore(testnet=True, dryrun=True)
sender_key = btctx_api.create_key()
signature = btctx_api.sign_unicode(sender_key, self.file_hash)
sender_address = btctx_api.get_address(sender_key)
self.mock_get.return_value = Response()
self.test_data_for_requests['headers'] = {
'sender-address': sender_address,
'signature': signature,
}
download_call_result = core.download(
self.test_url_address,
self.file_hash,
sender_key=sender_key,
btctx_api=btctx_api
)
expected_mock_calls = [call(
urljoin(self.test_url_address, '/api/files/' + self.file_hash),
**self.test_data_for_requests
)]
self.assertListEqual(
self.mock_get.call_args_list,
expected_mock_calls,
'In the download() function requests.get() calls are unexpected'
)
self.assertIsInstance(download_call_result, Response,
'Must return a response object')
def _setup_data_transfer_client(self, store_config, passive_port,
passive_bind, node_type, nat_type, wan_ip):
# Setup handlers for callbacks registered via the API.
handlers = {
"complete": self._transfer_complete_handlers,
"accept": self._transfer_request_handlers
}
wallet = BtcTxStore(testnet=False, dryrun=True)
wif = self.get_key()
node_id = address_to_node_id(wallet.get_address(wif))
#dht_node = SimDHT(node_id=node_id)
dht_node = self
self._data_transfer = FileTransfer(
net=Net(
net_type="direct",
node_type=node_type,
nat_type=nat_type,
dht_node=dht_node,
debug=1,
passive_port=passive_port,
passive_bind=passive_bind,
wan_ip=wan_ip
),
wif=wif,
store_config=store_config,
handlers=handlers
)
# Setup success callback values.
self._data_transfer.success_value = (self.sync_get_wan_ip(), self.port)
self.process_data_transfers()
def test_fail(self):
# register without auth headres fails
rv = self.app.get('/api/register/{0}'.format(addresses["eta"]))
self.assertEqual(rv.status_code, 401)
# register first because ping is lazy
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True,
usegmt=True)
message = app.config["ADDRESS"] + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
url = '/api/register/{0}'.format(address)
rv = self.app.get(url, headers=headers)
self.assertEqual(rv.status_code, 200)
# ping without auth headres fails
time.sleep(app.config["MAX_PING"])
rv = self.app.get('/api/ping/{0}'.format(address))
self.assertEqual(rv.status_code, 401)
# set height without auth headres fails
rv = self.app.get('/api/height/{0}/10'.format(addresses["eta"]))
self.assertEqual(rv.status_code, 401)
def callback():
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True, usegmt=True)
header_authorization = blockchain.sign_unicode(wif, "lalala-wrong")
farmer.authenticate(header_authorization, header_date)
class TestSignUnicode(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_sign_a(self):
wif = fixtures["wallet"]["wif"]
message = u"üöä"
address = self.api.get_address(wif)
sig = self.api.sign_unicode(wif, message)
valid = self.api.verify_signature_unicode(address, sig, message)
self.assertEqual(valid, True)
def test_sign_b(self):
wif = "cSuT2J14dYbe1zvB5z5WTXeRcMbj4tnoKssAK1ZQbnX5HtHfW3bi"
message = u"üöä"
address = self.api.get_address(wif)
sig = self.api.sign_unicode(wif, message)
valid = self.api.verify_signature_unicode(address, sig, message)
self.assertEqual(valid, True)
class TestSignData(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_sign_a(self):
wif = fixtures["wallet"]["wif"]
data = binascii.hexlify(b"testmessage")
address = self.api.get_address(wif)
sig = self.api.sign_data(wif, data)
valid = self.api.verify_signature(address, sig, data)
self.assertEqual(valid, True)
def test_sign_b(self):
wif = "cSuT2J14dYbe1zvB5z5WTXeRcMbj4tnoKssAK1ZQbnX5HtHfW3bi"
data = binascii.hexlify(b"testmessage")
address = self.api.get_address(wif)
sig = self.api.sign_data(wif, data)
valid = self.api.verify_signature(address, sig, data)
self.assertEqual(valid, True)
class TestAuth(unittest.TestCase):
def setUp(self):
self.btctxstore = BtcTxStore()
self.sender_wif = self.btctxstore.create_key()
self.sender = self.btctxstore.get_address(self.sender_wif)
recipient_wif = self.btctxstore.create_key()
self.recipient = self.btctxstore.get_address(recipient_wif)
def test_self_validates(self):
headers = storjcore.auth.create_headers(self.btctxstore,
self.recipient,
self.sender_wif)
self.assertTrue(storjcore.auth.verify_headers(self.btctxstore,
headers,
5, self.sender,
self.recipient))
def test_invalid_signature(self):
def callback():
headers = storjcore.auth.create_headers(self.btctxstore,
self.recipient,
self.sender_wif)
headers["Authorization"] = base64.b64encode(65 * b"x")
storjcore.auth.verify_headers(self.btctxstore, headers,
5, self.sender, self.recipient)
self.assertRaises(storjcore.auth.AuthError, callback)
def test_timeout_to_old(self):
def callback():
headers = storjcore.auth.create_headers(self.btctxstore,
self.recipient,
self.sender_wif)
time.sleep(5)
storjcore.auth.verify_headers(self.btctxstore, headers,
5, self.sender, self.recipient)
self.assertRaises(storjcore.auth.AuthError, callback)
@unittest.skip("TODO implement")
def test_timeout_to_young(self):
pass # FIXME how to test this?
class TestSignUnicode(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_sign_a(self):
wif = fixtures["wallet"]["wif"]
message = u"üöä"
address = self.api.get_address(wif)
sig = self.api.sign_unicode(wif, message)
valid = self.api.verify_signature_unicode(address, sig, message)
self.assertEqual(valid, True)
def test_sign_b(self):
wif = "cSuT2J14dYbe1zvB5z5WTXeRcMbj4tnoKssAK1ZQbnX5HtHfW3bi"
message = u"üöä"
address = self.api.get_address(wif)
sig = self.api.sign_unicode(wif, message)
valid = self.api.verify_signature_unicode(address, sig, message)
self.assertEqual(valid, True)
class TestSignData(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_sign_a(self):
wif = fixtures["wallet"]["wif"]
data = binascii.hexlify(b"testmessage")
address = self.api.get_address(wif)
sig = self.api.sign_data(wif, data)
valid = self.api.verify_signature(address, sig, data)
self.assertEqual(valid, True)
def test_sign_b(self):
wif = "cSuT2J14dYbe1zvB5z5WTXeRcMbj4tnoKssAK1ZQbnX5HtHfW3bi"
data = binascii.hexlify(b"testmessage")
address = self.api.get_address(wif)
sig = self.api.sign_data(wif, data)
valid = self.api.verify_signature(address, sig, data)
self.assertEqual(valid, True)
def test_authentication_success(self):
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True, usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
self.assertTrue(farmer.authenticate(header_authorization, header_date))
def test_authentication_success(self):
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True,
usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
self.assertTrue(farmer.authenticate(header_authorization, header_date))
def callback():
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
header_date = formatdate(timeval=mktime(datetime.now().timetuple())
, localtime=True, usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": None, "Authorization": header_authorization}
farmer.authenticate(headers)
class TestValidateAddressTestnet(unittest.TestCase):
def setUp(self):
self.testnet_api = BtcTxStore(dryrun=True, testnet=True)
self.mainnet_api = BtcTxStore(dryrun=True, testnet=False)
def test_valid_string(self):
address = 'migiScBNvVKYwEiCFhgBNGtZ87cdygtuSQ'
self.assertTrue(self.testnet_api.validate_address(address))
def test_valid_network(self):
address = self.testnet_api.get_address(self.testnet_api.create_key())
self.assertTrue(self.testnet_api.validate_address(address))
def test_invalid_network(self):
address = self.mainnet_api.get_address(self.mainnet_api.create_key())
self.assertFalse(self.testnet_api.validate_address(address))
def test_invalid_data(self):
self.assertFalse(self.testnet_api.validate_address("f483"))
def test_invalid_type(self):
self.assertFalse(self.testnet_api.validate_address(None))
class TestValidateAddressMainnet(unittest.TestCase):
def setUp(self):
self.testnet_api = BtcTxStore(dryrun=True, testnet=True)
self.mainnet_api = BtcTxStore(dryrun=True, testnet=False)
def test_valid_string(self):
address = '191GVvAaTRxLmz3rW3nU5jAV1rF186VxQc'
self.assertTrue(self.mainnet_api.validate_address(address))
def test_valid_network(self):
address = self.mainnet_api.get_address(self.mainnet_api.create_key())
self.assertTrue(self.mainnet_api.validate_address(address))
def test_invalid_network(self):
address = self.testnet_api.get_address(self.testnet_api.create_key())
self.assertFalse(self.mainnet_api.validate_address(address))
def test_invalid_data(self):
self.assertFalse(self.mainnet_api.validate_address("f483"))
def test_invalid_type(self):
self.assertFalse(self.mainnet_api.validate_address(None))
class TestValidateAddressMainnet(unittest.TestCase):
def setUp(self):
self.testnet_api = BtcTxStore(dryrun=True, testnet=True)
self.mainnet_api = BtcTxStore(dryrun=True, testnet=False)
def test_valid_string(self):
address = '191GVvAaTRxLmz3rW3nU5jAV1rF186VxQc'
self.assertTrue(self.mainnet_api.validate_address(address))
def test_valid_network(self):
address = self.mainnet_api.get_address(self.mainnet_api.create_key())
self.assertTrue(self.mainnet_api.validate_address(address))
def test_invalid_network(self):
address = self.testnet_api.get_address(self.testnet_api.create_key())
self.assertFalse(self.mainnet_api.validate_address(address))
def test_invalid_data(self):
self.assertFalse(self.mainnet_api.validate_address("f483"))
def test_invalid_type(self):
self.assertFalse(self.mainnet_api.validate_address(None))
def callback():
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
header_date = formatdate(timeval=mktime(
datetime.now().timetuple()),
localtime=True,
usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": None, "Authorization": header_authorization}
farmer.authenticate(headers)
class TestValidateAddressTestnet(unittest.TestCase):
def setUp(self):
self.testnet_api = BtcTxStore(dryrun=True, testnet=True)
self.mainnet_api = BtcTxStore(dryrun=True, testnet=False)
def test_valid_string(self):
address = 'migiScBNvVKYwEiCFhgBNGtZ87cdygtuSQ'
self.assertTrue(self.testnet_api.validate_address(address))
def test_valid_network(self):
address = self.testnet_api.get_address(self.testnet_api.create_key())
self.assertTrue(self.testnet_api.validate_address(address))
def test_invalid_network(self):
address = self.mainnet_api.get_address(self.mainnet_api.create_key())
self.assertFalse(self.testnet_api.validate_address(address))
def test_invalid_data(self):
self.assertFalse(self.testnet_api.validate_address("f483"))
def test_invalid_type(self):
self.assertFalse(self.testnet_api.validate_address(None))
def callback():
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
timeout = farmer.get_server_authentication_timeout()
date = datetime.now() - timedelta(seconds=timeout)
header_date = formatdate(timeval=mktime(date.timetuple()),
localtime=True, usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
farmer.authenticate(header_authorization, header_date)
def callback():
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
timeout = farmer.get_server_authentication_timeout()
date = datetime.now() - timedelta(seconds=timeout)
header_date = formatdate(timeval=mktime(date.timetuple()),
localtime=True,
usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
farmer.authenticate(header_authorization, header_date)
def test_authentication_timeout_future_success(self):
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
timeout = farmer.get_server_authentication_timeout() - 5
date = datetime.now() + timedelta(seconds=timeout)
header_date = formatdate(timeval=mktime(date.timetuple()),
localtime=True, usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
self.assertTrue(farmer.authenticate(headers))
def test_success(self):
# create header date and authorization signature
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True, usegmt=True)
message = app.config["ADDRESS"] + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
url = '/api/register/{0}'.format(address)
rv = self.app.get(url, headers=headers)
data = json.loads(rv.data.decode("utf-8"))
self.assertEqual(address, data["btc_addr"])
self.assertEqual(rv.status_code, 200)
def test_authentication_timeout_future_success(self):
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
farmer = Farmer(address)
timeout = farmer.get_server_authentication_timeout() - 5
date = datetime.now() + timedelta(seconds=timeout)
header_date = formatdate(timeval=mktime(date.timetuple()),
localtime=True,
usegmt=True)
message = farmer.get_server_address() + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
self.assertTrue(farmer.authenticate(headers))
def test_success(self):
# create header date and authorization signature
blockchain = BtcTxStore()
wif = blockchain.create_key()
address = blockchain.get_address(wif)
header_date = formatdate(timeval=mktime(datetime.now().timetuple()),
localtime=True,
usegmt=True)
message = app.config["ADDRESS"] + " " + header_date
header_authorization = blockchain.sign_unicode(wif, message)
headers = {"Date": header_date, "Authorization": header_authorization}
url = '/api/register/{0}'.format(address)
rv = self.app.get(url, headers=headers)
data = json.loads(rv.data.decode("utf-8"))
self.assertEqual(address, data["btc_addr"])
self.assertEqual(rv.status_code, 200)
class TestVerifySignature(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_verify_positive(self):
_fixtures = fixtures["verify_signature"]["positive"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessage")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, True)
def test_verify_incorrect_address(self):
_fixtures = fixtures["verify_signature"]["incorrect_address"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessage")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, False)
def test_verify_incorrect_signature(self):
_fixtures = fixtures["verify_signature"]["incorrect_signature"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessage")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, False)
def test_verify_incorrect_data(self):
_fixtures = fixtures["verify_signature"]["incorrect_data"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessagee")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, False)
def test_verify_signature_params(self):
wif = "cSuT2J14dYbe1zvB5z5WTXeRcMbj4tnoKssAK1ZQbnX5HtHfW3bi"
data = binascii.hexlify(b"testmessage")
address = self.api.get_address(wif)
sig = "///////////////////////////////////////////////////////////////////////////////////////="
self.assertFalse(self.api.verify_signature(address, sig, data))
class TestVerifySignature(unittest.TestCase):
def setUp(self):
self.api = BtcTxStore(dryrun=True, testnet=True)
def test_verify_positive(self):
_fixtures = fixtures["verify_signature"]["positive"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessage")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, True)
def test_verify_incorrect_address(self):
_fixtures = fixtures["verify_signature"]["incorrect_address"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessage")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, False)
def test_verify_incorrect_signature(self):
_fixtures = fixtures["verify_signature"]["incorrect_signature"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessage")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, False)
def test_verify_incorrect_data(self):
_fixtures = fixtures["verify_signature"]["incorrect_data"]
address = _fixtures["address"]
signature = _fixtures["signature"]
data = binascii.hexlify(b"testmessagee")
result = self.api.verify_signature(address, signature, data)
self.assertEqual(result, False)
def test_verify_signature_params(self):
wif = "cSuT2J14dYbe1zvB5z5WTXeRcMbj4tnoKssAK1ZQbnX5HtHfW3bi"
data = binascii.hexlify(b"testmessage")
address = self.api.get_address(wif)
sig = ("/////////////////////////////////////////"
"//////////////////////////////////////////////=")
self.assertFalse(self.api.verify_signature(address, sig, data))
def mph_status(assets=None):
with etc.database_lock:
verify.status_input(assets)
btctxstore = BtcTxStore(testnet=etc.testnet)
wif = lib.load_wif()
address = btctxstore.get_address(wif)
message = util.b2h(os.urandom(32))
signature = btctxstore.sign_unicode(wif, message)
if isinstance(signature, bytes): # XXX update btctxstore instead !!!
signature = signature.decode("utf-8")
return {
"funds": {
"address": address,
"message": message,
"signature": signature,
"liquidity": lib.get_hub_liquidity(assets=assets),
},
"current_terms": lib.get_terms(assets=assets),
"connections": lib.get_connections_status(assets=assets)
}
def verify_signature(msg, wif, node_id=None): # FIXME use read instead
assert(isinstance(msg, OrderedDict))
if u"signature" not in msg:
return 0
msg = msg.copy() # work on a copy for thread saftey
sig = msg.pop("signature")
# Use our address.
api = BtcTxStore(testnet=False, dryrun=True)
try:
if node_id is None:
address = api.get_address(wif)
ret = api.verify_signature_unicode(address, sig, str(msg))
else:
address = node_id_to_address(node_id)
ret = api.verify_signature_unicode(address, sig, str(msg))
except TypeError:
return 0
return ret
class FarmerUpTime(unittest.TestCase):
def setUp(self):
app.config["SKIP_AUTHENTICATION"] = True # monkey patch
app.config["DISABLE_CACHING"] = True
self.btctxstore = BtcTxStore()
db.create_all()
def tearDown(self):
db.session.remove()
db.drop_all()
def test_register(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 100
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
# reg_time = max online time -> 100% uptime
delta = timedelta(minutes=app.config["ONLINE_TIME"])
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": delta.seconds,
"uptime": 100
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
# reg_time = 2 * max online time -> 50% uptime
delta = timedelta(minutes=(2 * app.config["ONLINE_TIME"]))
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.ping()
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 50
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_ping_100(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# lastest ping for 100%
delta = timedelta(minutes=app.config["ONLINE_TIME"])
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.ping()
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 100
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_ping_50(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# lastest ping for 100%
delta = timedelta(minutes=(2 * app.config["ONLINE_TIME"]))
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.ping()
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 50
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_ping_25(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# ping to late -> 50%
delta = timedelta(minutes=(4 * app.config["ONLINE_TIME"]))
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.ping()
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 25
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_ping_days(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# ping to late -> 50%
delta = timedelta(days=2)
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.uptime = 86401 # 1 / 2 days farmer was online
farmer.ping()
test_json = {
"height": 0,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 50
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_height_100(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# lastest ping for 100%
delta = timedelta(minutes=app.config["ONLINE_TIME"])
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.set_height(100)
test_json = {
"height": 100,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 100
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_height_50(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# lastest ping for 100%
delta = timedelta(minutes=(2 * app.config["ONLINE_TIME"]))
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.set_height(50)
test_json = {
"height": 50,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 50
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
def test_height_25(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
#ping to late -> 50%
delta = timedelta(minutes=(4 * app.config["ONLINE_TIME"]))
farmer.last_seen = datetime.utcnow() - delta
farmer.reg_time = datetime.utcnow() - delta
farmer.set_height(25)
test_json = {
"height": 25,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 25
}
call_payload = json.loads(farmer.to_json())
call_payload["uptime"] = round(call_payload["uptime"])
self.assertEqual(test_json, call_payload)
# store data in blockchain as nulldata output (max 40bytes)
data = binascii.hexlify(b"example_data")
txid = api.store_nulldata(data, wifs)
# Show current transaction id
print("Current Transaction ID: {}".format(txid))
# Now, retrieve data based on transaction id
hexnulldata = api.retrieve_nulldata(txid)
print("Retrieved Data: {}".format(hexnulldata))
# create new private key
wif = api.create_key()
# get private key address
address = api.get_address(wif)
# hexlify messagetext
data = binascii.hexlify(b"messagetext")
# sign data with private key
signature = api.sign_data(wif, data)
print("signature:", signature)
# verify signature (no public or private key needed)
isvalid = api.verify_signature(address, signature, data)
print("valid signature" if isvalid else "invalid signature")
class FarmerTest(unittest.TestCase):
def setUp(self):
app.config["SKIP_AUTHENTICATION"] = True # monkey patch
app.config["DISABLE_CACHING"] = True
self.btctxstore = BtcTxStore()
self.bad_addr = 'notvalidaddress'
db.create_all()
def tearDown(self):
db.session.remove()
db.drop_all()
def test_repr(self):
farmer = Farmer('191GVvAaTRxLmz3rW3nU5jAV1rF186VxQc')
ans = "<Farmer BTC Address: '191GVvAaTRxLmz3rW3nU5jAV1rF186VxQc'>"
self.assertEqual(repr(farmer), ans)
def test_sha256(self):
an = 'c059c8035bbd74aa81f4c787c39390b57b974ec9af25a7248c46a3ebfe0f9dc8'
self.assertEqual(sha256("storj"), an)
self.assertNotEqual(sha256("not storj"), an)
def test_register(self):
# test success
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer1 = Farmer(btc_addr)
self.assertFalse(farmer1.exists())
farmer1.register()
self.assertTrue(farmer1.exists())
# test duplicate error
self.assertRaises(LookupError, farmer1.register)
def callback_a():
Farmer(self.bad_addr)
self.assertRaises(ValueError, callback_a)
def test_ping(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
# test ping before registration
self.assertRaises(LookupError, farmer.ping)
# register farmer
farmer.register()
# get register time, and make sure the ping work
register_time = farmer.last_seen
# ping faster than max_ping would be ignored
time.sleep(app.config["MAX_PING"] + 1)
farmer.ping() # update last seen
ping_time = farmer.last_seen
self.assertTrue(register_time < ping_time)
def test_ping_time_limit(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
register_time = farmer.last_seen
time.sleep(2)
farmer.ping()
# should still be around 0
delta_seconds = int((farmer.last_seen - register_time).seconds)
self.assertEqual(delta_seconds, 0)
def test_height(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
# set height and check function output
self.assertEqual(farmer.height, 0)
self.assertEqual(farmer.set_height(5), 5)
self.assertEqual(farmer.height, 5)
# check the db object as well
farmer2 = farmer.lookup()
self.assertEqual(farmer2.height, 5)
def test_audit(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
# test audit before registration
self.assertRaises(LookupError, farmer.audit)
# register farmer
farmer.register()
# get register time, and make sure the ping work
register_time = farmer.last_seen
# ping faster than max_ping would be ignored
time.sleep(app.config["MAX_PING"] + 1)
farmer.audit()
ping_time = farmer.last_seen
self.assertTrue(register_time < ping_time)
def test_to_json(self):
btc_addr = self.btctxstore.get_address(
self.btctxstore.get_key(self.btctxstore.create_wallet()))
farmer = Farmer(btc_addr)
farmer.register()
farmer.ping()
farmer.set_height(50)
test_json = {
"height": 50,
"btc_addr": btc_addr,
'payout_addr': btc_addr,
"last_seen": 0,
"uptime": 100
}
call_payload = json.loads(farmer.to_json())
self.assertEqual(test_json, call_payload)
#!/usr/bin/env python
# coding: utf-8
# Copyright (c) 2015 Fabian Barkhau <*****@*****.**>
# License: MIT (see LICENSE file)
from __future__ import print_function
from __future__ import unicode_literals
from btctxstore import BtcTxStore
import time
import cProfile
from pstats import Stats
api = BtcTxStore(testnet=True, dryrun=True) # use testing setup for example
wif = api.create_key() # create new private key
address = api.get_address(wif) # get private key address
message = "Signed ünicöde message."
signature = api.sign_unicode(wif, message)
profile = cProfile.Profile()
profile.enable()
begin = time.time()
for i in range(10):
assert (api.verify_signature_unicode(address, signature, message))
end = time.time()
stats = Stats(profile)
stats.strip_dirs()
stats.sort_stats('cumtime')
stats.print_stats()
class FileTransfer:
def __init__(self, net, wif=None, store_config=None, handlers=None):
# Accept direct connections.
self.net = net
# Returned by callbacks.
self.success_value = ("127.0.0.1", 7777)
# Used for signing messages.
self.wallet = BtcTxStore(testnet=False, dryrun=True)
self.wif = wif or self.wallet.create_key()
# Where will the data be stored?
self.store_config = store_config
assert(len(list(store_config)))
# Handlers for certain events.
self.handlers = handlers
if self.handlers is None:
self.handlers = {}
if "complete" not in self.handlers:
self.handlers["complete"] = []
if "accept" not in self.handlers:
self.handlers["accept"] = []
# Start networking.
if not self.net.is_net_started:
self.net.start()
# Dict of data requests: [contract_id] > contract
self.contracts = {}
# List of Sock objects returned from UNL.connect.
self.cons = []
# Dict of defers for contracts: [contract_id] > defer
self.defers = {}
# Three-way handshake status for contracts: [contract_id] > state
self.handshake = {}
# All contracts associated with this connection.
# [con] > [contract_id] > con_info
self.con_info = {}
# File transfer currently active on connection.
# [con] > contract_id
self.con_transfer = {}
# List of active downloads.
# (Never try to download multiple copies of the same thing at once.)
self.downloading = {}
# Lock threads.
self.mutex = Lock()
def get_their_unl(self, contract):
if self.net.unl == pyp2p.unl.UNL(value=contract["dest_unl"]):
their_unl = contract["src_unl"]
else:
their_unl = contract["dest_unl"]
return their_unl
def get_node_id_from_unl(self, unl):
unl = pyp2p.unl.UNL(value=unl).deconstruct()
return unl["node_id"]
def is_queued(self, con=None):
if con is not None:
if con not in self.con_info:
return 0
if con is None:
con_list = list(self.con_info)
else:
con_list = [con]
for con in con_list:
for contract_id in list(self.con_info[con]):
con_info = self.con_info[con][contract_id]
if con_info["remaining"]:
return 1
return 0
def cleanup_transfers(self, con, contract_id):
# Cleanup downloading.
contract = self.contracts[contract_id]
if contract["data_id"] in self.downloading:
if contract["direction"] == "receive":
del self.downloading[contract["data_id"]]
# Cleanup handshakes.
if contract_id in self.handshake:
del self.handshake[contract_id]
# Cleanup defers.
if contract_id in self.defers:
del self.defers[contract_id]
# Cleanup con transfers.
if con in self.con_transfer:
del self.con_transfer[con]
# Cleanup con_info.
if con in self.con_info:
del self.con_info[con]
# Cleanup contracts.
if contract_id in self.contracts:
del self.contracts[contract_id]
def queue_next_transfer(self, con):
_log.debug("Queing next transfer")
for contract_id in list(self.con_info[con]):
con_info = self.con_info[con][contract_id]
if con_info["remaining"]:
self.con_transfer[con] = contract_id
con.send(contract_id, send_all=1)
return
# Mark end of transfers.
self.con_transfer[con] = u"0" * 64
def save_contract(self, contract):
# Record contract details.
contract_id = self.contract_id(contract)
self.contracts[contract_id] = contract
return contract_id
def send_msg(self, dict_obj, unl):
node_id = self.net.unl.deconstruct(unl)["node_id"]
msg = json.dumps(dict_obj, ensure_ascii=True)
self.net.dht_node.relay_message(
node_id,
msg
)
def contract_id(self, contract):
if sys.version_info >= (3, 0, 0):
contract = str(contract).encode("ascii")
else:
contract = str(contract)
return hashlib.sha256(contract).hexdigest()
def sign_contract(self, contract):
if sys.version_info >= (3, 0, 0):
msg = str(contract).encode("ascii")
else:
msg = str(contract)
msg = binascii.hexlify(msg).decode("utf-8")
sig = self.wallet.sign_data(self.wif, msg)
if sys.version_info >= (3, 0, 0):
contract[u"signature"] = sig.decode("utf-8")
else:
contract[u"signature"] = unicode(sig)
return contract
def is_valid_contract_sig(self, contract, node_id=None):
sig = contract[u"signature"][:]
del contract[u"signature"]
if sys.version_info >= (3, 0, 0):
msg = str(contract).encode("ascii")
else:
msg = str(contract)
# Use our address.
msg = binascii.hexlify(msg).decode("utf-8")
if node_id is None:
address = self.wallet.get_address(self.wif)
ret = self.wallet.verify_signature(address, sig, msg)
else:
# Use their node ID: try testnet.
address = b2a_hashed_base58(b'o' + node_id)
ret = self.wallet.verify_signature(address, sig, msg)
if not ret:
# Use their node ID: try mainnet.
address = b2a_hashed_base58(b'\0' + node_id)
ret = self.wallet.verify_signature(address, sig, msg)
# Move sig back.
contract[u"signature"] = sig[:]
return ret
def simple_data_request(self, data_id, node_unl, direction):
file_size = 0
if direction == u"send":
action = u"upload"
else:
action = u"download"
return self.data_request(action, data_id, file_size, node_unl)
def data_request(self, action, data_id, file_size, node_unl):
"""
Action = put (upload), get (download.)
"""
_log.debug("In data request function")
# Who is hosting this data?
if action == "upload":
# We store this data.
direction = u"send"
host_unl = self.net.unl.value
assert(storage.manager.find(self.store_config, data_id) is not None)
else:
# They store the data.
direction = u"receive"
host_unl = node_unl
if data_id in self.downloading:
raise Exception("Already trying to download this.")
# Encoding.
if sys.version_info >= (3, 0, 0):
if type(data_id) == bytes:
data_id = data_id.decode("utf-8")
if type(host_unl) == bytes:
host_unl = host_unl.decode("utf-8")
if type(node_unl) == bytes:
node_unl = node_unl.decode("utf-8")
else:
if type(data_id) == str:
data_id = unicode(data_id)
if type(host_unl) == str:
host_unl = unicode(host_unl)
if type(node_unl) == str:
node_unl = unicode(node_unl)
# Create contract.
contract = OrderedDict({
u"status": u"SYN",
u"direction": direction,
u"data_id": data_id,
u"file_size": file_size,
u"host_unl": host_unl,
u"dest_unl": node_unl,
u"src_unl": self.net.unl.value,
})
# Sign contract.
contract = self.sign_contract(contract)
# Route contract.
contract_id = self.save_contract(contract)
self.send_msg(contract, node_unl)
_log.debug("Sending data request")
# Update handshake.
self.handshake[contract_id] = {
u"state": u"SYN",
u"timestamp": time.time()
}
# For async code.
d = defer.Deferred()
self.defers[contract_id] = d
# Return defer for async code.
return d
def get_con_by_contract_id(self, needle):
for con in list(self.con_info):
for contract_id in list(self.con_info[con]):
if contract_id == needle:
return con
return None
def remove_file_from_storage(self, data_id):
storage.manager.remove(self.store_config, data_id)
def move_file_to_storage(self, path):
with open(path, "rb") as shard:
storage.manager.add(self.store_config, shard)
return {
"file_size": storage.shard.get_size(shard),
"data_id": storage.shard.get_id(shard)
}
def get_data_chunk(self, data_id, position, chunk_size=1048576):
path = storage.manager.find(self.store_config, data_id)
buf = b""
with open(path, "rb") as fp:
fp.seek(position, 0)
buf = fp.read(chunk_size)
return buf
def save_data_chunk(self, data_id, chunk):
_log.debug("Saving data chunk for " + str(data_id))
_log.debug("of size + " + str(len(chunk)))
assert(data_id in self.downloading)
# Find temp file path.
path = self.downloading[data_id]
_log.debug(path)
with open(path, "ab") as fp:
fp.write(chunk)
class Client(object):
def __init__(self, url=common.DEFAULT_URL, debug=False,
max_size=common.DEFAULT_MAX_SIZE,
store_path=common.DEFAULT_STORE_PATH,
config_path=common.DEFAULT_CONFIG_PATH,
set_master_secret=None, set_payout_address=None,
connection_retry_limit=common.DEFAULT_CONNECTION_RETRY_LIMIT,
connection_retry_delay=common.DEFAULT_CONNECTION_RETRY_DELAY):
# FIXME validate master_secret
self.url = url
self.config = None # lazy
self.messanger = None # lazy
self.debug = debug # TODO validate
self.retry_limit = deserialize.positive_integer(connection_retry_limit)
self.retry_delay = deserialize.positive_integer(connection_retry_delay)
self.max_size = deserialize.byte_count(max_size)
# paths
self.config_path = os.path.realpath(config_path) # TODO validate
self._ensure_path_exists(os.path.dirname(self.config_path))
self.store_path = os.path.realpath(store_path)
self._ensure_path_exists(self.store_path)
# validate payout address
self.btctxstore = BtcTxStore()
if set_payout_address and (not self.btctxstore.validate_address(set_payout_address)):
raise exceptions.InvalidAddress(set_payout_address)
self._initialize_config(set_master_secret, set_payout_address)
def _initialize_config(self, set_master_secret, set_payout_address):
if os.path.exists(self.config_path):
self._load_config()
else: # initialize config
if not set_master_secret: # create random master secret
secret_data = base64.b64encode(os.urandom(256))
set_master_secret = secret_data.decode('utf-8')
if not set_payout_address: # use root address if not given
set_payout_address = self._get_root_address(set_master_secret)
if set_master_secret or set_payout_address:
self.config = {
"version": __version__,
"master_secret": set_master_secret,
"payout_address": set_payout_address,
}
config.save(self.config_path, self.config)
def _ensure_path_exists(self, path):
if not os.path.exists(path):
os.makedirs(path)
def _load_config(self):
if self.config is None:
if not os.path.exists(self.config_path):
raise exceptions.ConfigNotFound(self.config_path)
self.config = config.load(self.config_path)
def _get_root_wif(self, master_secret):
master_secret = base64.b64decode(master_secret)
hwif = self.btctxstore.create_wallet(master_secret=master_secret)
return self.btctxstore.get_key(hwif)
def _get_root_address(self, master_secret):
wif = self._get_root_wif(master_secret)
return self.btctxstore.get_address(wif)
def _init_messanger(self):
if self.messanger is None:
wif = self._get_root_wif(self.config["master_secret"])
self.messanger = messaging.Messaging(self.url, wif,
self.retry_limit,
self.retry_delay)
def version(self):
print(__version__)
return __version__
def register(self):
"""Attempt to register the config address."""
self._init_messanger()
registered = self.messanger.register(self.config["payout_address"])
auth_addr = self.messanger.auth_address()
if registered:
print("Address {0} now registered on {1}.".format(auth_addr,
self.url))
else:
print("Failed to register address {0} on {1}.".format(auth_addr,
self.url))
return True
def show_config(self):
"""Display saved config."""
print(json.dumps(self.config, indent=2))
return self.config
def ping(self):
"""Attempt keep-alive with the server."""
self._init_messanger()
print("Pinging {0} with address {1}.".format(
self.messanger.server_url(), self.messanger.auth_address()))
self.messanger.ping()
return True
def poll(self, register_address=False, delay=common.DEFAULT_DELAY,
limit=None):
"""TODO doc string"""
stop_time = _now() + timedelta(seconds=int(limit)) if limit else None
if register_address:
self.register()
while True:
self.ping()
if stop_time and _now() >= stop_time:
return True
time.sleep(int(delay))
def build(self, cleanup=False, rebuild=False,
set_height_interval=common.DEFAULT_SET_HEIGHT_INTERVAL):
"""TODO doc string"""
self._init_messanger()
def _on_generate_shard(height, seed, file_hash):
first = height == 1
set_height = (height % set_height_interval) == 0
last = (int(self.max_size / common.SHARD_SIZE) + 1) == height
if first or set_height or last:
self.messanger.height(height)
bldr = builder.Builder(self.config["payout_address"],
common.SHARD_SIZE, self.max_size,
debug=self.debug,
on_generate_shard=_on_generate_shard)
generated = bldr.build(self.store_path, cleanup=cleanup,
rebuild=rebuild)
height = len(generated)
self.messanger.height(height)
return generated
#!/usr/bin/env python
# coding: utf-8
# Copyright (c) 2015 Fabian Barkhau <*****@*****.**>
# License: MIT (see LICENSE file)
from __future__ import print_function
from __future__ import unicode_literals
from btctxstore import BtcTxStore
import time
import cProfile
from pstats import Stats
api = BtcTxStore(testnet=True, dryrun=True) # use testing setup for example
wif = api.create_key() # create new private key
address = api.get_address(wif) # get private key address
message = "Signed ünicöde message."
signature = api.sign_unicode(wif, message)
profile = cProfile.Profile()
profile.enable()
begin = time.time()
for i in range(10):
assert(api.verify_signature_unicode(address, signature, message))
end = time.time()
stats = Stats(profile)
stats.strip_dirs()
stats.sort_stats('cumtime')
def test_bandwidth_test(self):
# Alice sample node.
alice_wallet = BtcTxStore(testnet=False, dryrun=True)
alice_wif = alice_wallet.create_key()
alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif))
alice_dht = pyp2p.dht_msg.DHT(
node_id=alice_node_id,
networking=0
)
alice_transfer = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63600,
debug=1,
wan_ip="8.8.8.8",
dht_node=alice_dht,
),
wif=alice_wif,
store_config={tempfile.mkdtemp(): None}
)
_log.debug("Alice UNL")
_log.debug(alice_transfer.net.unl.value)
# Bob sample node.
bob_wallet = BtcTxStore(testnet=False, dryrun=True)
bob_wif = bob_wallet.create_key()
bob_node_id = address_to_node_id(bob_wallet.get_address(bob_wif))
bob_dht = pyp2p.dht_msg.DHT(
node_id=bob_node_id,
networking=0
)
bob_transfer = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63601,
debug=1,
wan_ip="8.8.8.8",
dht_node=bob_dht
),
wif=bob_wif,
store_config={tempfile.mkdtemp(): None}
)
# Link DHT nodes.
alice_dht.add_relay_link(bob_dht)
bob_dht.add_relay_link(alice_dht)
_log.debug("Bob UNL")
_log.debug(bob_transfer.net.unl.value)
# Show bandwidth.
def show_bandwidth(results):
global test_success
test_success = 1
_log.debug(results)
# Test bandwidth between Alice and Bob.
bob_test = BandwidthTest(bob_wif, bob_transfer, bob_dht, 0)
alice_test = BandwidthTest(alice_wif, alice_transfer, alice_dht, 0)
d = alice_test.start(bob_transfer.net.unl.value)
d.addCallback(show_bandwidth)
# Main event loop.
# and not test_success
end_time = time.time() + 60
while alice_test.active_test is not None and time.time() < end_time:
for client in [alice_transfer, bob_transfer]:
process_transfers(client)
time.sleep(0.002)
# End net.
for client in [alice_transfer, bob_transfer]:
client.net.stop()
self.assertTrue(test_success == 1)
# Find temp file path.
path = self.downloading[data_id]
_log.debug(path)
with open(path, "ab") as fp:
fp.write(chunk)
if __name__ == "__main__":
from crochet import setup
setup()
# Alice sample node.
alice_wallet = BtcTxStore(testnet=False, dryrun=True)
alice_wif = alice_wallet.create_key()
alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif))
# print(type(alice_node_id))
alice_dht_node = pyp2p.dht_msg.DHT(node_id=alice_node_id)
# print(alice_dht_node.get_id())
alice_dht_node = storjnode.network.Node(
alice_wif, bootstrap_nodes=[("240.0.0.0", 1337)],
disable_data_transfer=True
)
alice = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
class FileTransfer:
def __init__(self, net, wif=None, store_config=None, handlers=None):
# Accept direct connections.
self.net = net
# Returned by callbacks.
self.success_value = ("127.0.0.1", 7777)
# Used for signing messages.
self.wallet = BtcTxStore(testnet=True, dryrun=True)
self.wif = wif or self.wallet.create_key()
# Where will the data be stored?
self.store_config = store_config
assert (len(list(store_config)))
# Handlers for certain events.
self.handlers = handlers
# Start networking.
if not self.net.is_net_started:
self.net.start()
# Dict of data requests.
self.contracts = {}
# Dict of defers for contracts.
self.defers = {}
# Three-way handshake status for contracts.
self.handshake = {}
# All contracts associated with this connection.
self.con_info = {}
# File transfer currently active on connection.
self.con_transfer = {}
# List of active downloads.
# (Never try to download multiple copies of the same thing at once.)
self.downloading = {}
def get_their_unl(self, contract):
if self.net.unl == pyp2p.unl.UNL(value=contract["dest_unl"]):
their_unl = contract["src_unl"]
else:
their_unl = contract["dest_unl"]
return their_unl
def is_queued(self, con=None):
if con is not None:
if con not in self.con_info:
return 0
if con is None:
con_list = list(self.con_info)
else:
con_list = [con]
for con in con_list:
for contract_id in list(self.con_info[con]):
con_info = self.con_info[con][contract_id]
if con_info["remaining"]:
return 1
return 0
def cleanup_transfers(self, con):
# Close con - there's nothing left to download.
if not self.is_queued(con):
# Cleanup con transfers.
if con in self.con_transfer:
del self.con_transfer[con]
# Cleanup con_info.
if con in self.con_info:
del self.con_info[con]
# Todo: cleanup contract + handshake state.
def queue_next_transfer(self, con):
_log.debug("Queing next transfer")
for contract_id in list(self.con_info[con]):
con_info = self.con_info[con][contract_id]
if con_info["remaining"]:
self.con_transfer[con] = contract_id
con.send(contract_id, send_all=1)
return
# Mark end of transfers.
self.con_transfer[con] = u"0" * 64
def is_valid_syn(self, msg):
# List of expected fields.
syn_schema = (u"status", u"direction", u"data_id", u"file_size",
u"host_unl", u"dest_unl", u"src_unl", u"signature")
# Check all fields exist.
if not all(key in msg for key in syn_schema):
_log.debug("Missing required key.")
return 0
# Check SYN size.
if len(msg) > 5242880: # 5 MB.
_log.debug("SYN is too big")
return 0
# Check direction is valid.
direction_tuple = (u"send", u"receive")
if msg[u"direction"] not in direction_tuple:
_log.debug("Missing required direction tuple.")
return 0
# Check the UNLs are valid.
unl_tuple = (u"host_unl", u"dest_unl", u"src_unl")
for unl_key in unl_tuple:
if not pyp2p.unl.is_valid_unl(msg[unl_key]):
_log.debug("Invalid UNL for " + unl_key)
_log.debug(msg[unl_key])
return 0
# Check file size.
file_size_type = type(msg[u"file_size"])
if sys.version_info >= (3, 0, 0):
expr = file_size_type != int
else:
expr = file_size_type != int and file_size_type != long
if expr:
_log.debug("File size validation failed")
_log.debug(type(msg[u"file_size"]))
return 0
# Are we the host?
if self.net.unl == pyp2p.unl.UNL(value=msg[u"host_unl"]):
# Then check we have this file.
path = storage.manager.find(self.store_config, msg[u"data_id"])
if path is None:
_log.debug("Failed to find file we're uploading")
return 0
else:
# Do we already have this file?
path = storage.manager.find(self.store_config, msg[u"data_id"])
if path is not None:
_log.debug("Attempting to download file we already have")
return 0
# Are we already trying to download this?
if msg[u"data_id"] in self.downloading:
_log.debug("We're already trying to download this")
return 0
return 1
def protocol(self, msg):
msg = json.loads(msg, object_pairs_hook=OrderedDict)
# Associate TCP con with contract.
def success_wrapper(self, contract_id, host_unl):
def success(con):
with mutex:
_log.debug("IN SUCCESS CALLBACK")
_log.debug("Success() contract_id = " + str(contract_id))
# Associate TCP con with contract.
contract = self.contracts[contract_id]
file_size = contract["file_size"]
# Store con association.
if con not in self.con_info:
self.con_info[con] = {}
# Associate contract with con.
if contract_id not in self.con_info[con]:
self.con_info[con][contract_id] = {
"contract_id": contract_id,
"remaining": 350, # Tree fiddy.
"file_size": file_size,
"file_size_buf": b""
}
# Record download state.
data_id = contract["data_id"]
if self.net.unl != pyp2p.unl.UNL(value=host_unl):
_log.debug("Success: download")
fp, self.downloading[data_id] = tempfile.mkstemp()
else:
# Set initial upload for this con.
_log.debug("Success: upload")
# Queue first transfer.
their_unl = self.get_their_unl(contract)
is_master = self.net.unl.is_master(their_unl)
_log.debug("Is master = " + str(is_master))
if con not in self.con_transfer:
if is_master:
# A transfer to queue processing.
self.queue_next_transfer(con)
else:
# A transfer to receive (unknown.)
self.con_transfer[con] = u""
else:
if self.con_transfer[con] == u"0" * 64:
if is_master:
self.queue_next_transfer(con)
else:
self.con_transfer[con] = u""
return success
# Sanity checking.
if u"status" not in msg:
return
# Accept data request.
if msg[u"status"] == u"SYN":
# Check syn is valid.
if not self.is_valid_syn(msg):
_log.debug("SYN: invalid syn.")
return
# Save contract.
contract_id = self.contract_id(msg)
self.save_contract(msg)
self.handshake[contract_id] = {
"state": u"SYN-ACK",
"timestamp": time.time()
}
# Create reply.
reply = OrderedDict({
u"status": u"SYN-ACK",
u"syn": msg,
})
# Sign reply.
reply = self.sign_contract(reply)
# Save reply.
self.send_msg(reply, msg[u"src_unl"])
_log.debug("SYN")
# Confirm accept and make connection if needed.
if msg[u"status"] == u"SYN-ACK":
# Valid syn-ack?
if u"syn" not in msg:
_log.debug("SYN-ACK: syn not in msg.")
return
# Is this a reply to our SYN?
contract_id = self.contract_id(msg[u"syn"])
if contract_id not in self.contracts:
_log.debug("--------------")
_log.debug(msg)
_log.debug("--------------")
_log.debug(self.contracts)
_log.debug("--------------")
_log.debug("SYN-ACK: contract not found.")
return
# Check syn is valid.
if not self.is_valid_syn(msg[u"syn"]):
_log.debug("SYN-ACK: invalid syn.")
return
# Did I sign this?
if not self.is_valid_contract_sig(msg[u"syn"]):
_log.debug("SYN-ACK: sig is invalid.")
return
# Update handshake.
contract = self.contracts[contract_id]
self.handshake[contract_id] = {
"state": u"ACK",
"timestamp": time.time()
}
# Create reply contract.
reply = OrderedDict({u"status": u"ACK", u"syn_ack": msg})
# Sign reply.
reply = self.sign_contract(reply)
# Try make TCP con.
self.net.unl.connect(contract["dest_unl"], {
"success":
success_wrapper(self, contract_id, contract["host_unl"])
},
force_master=0,
nonce=contract_id)
# Send reply.
self.send_msg(reply, msg[u"syn"][u"dest_unl"])
_log.debug("SYN-ACK")
if msg[u"status"] == u"ACK":
# Valid ack.
if u"syn_ack" not in msg:
_log.debug("ACK: syn_ack not in msg.")
return
if u"syn" not in msg[u"syn_ack"]:
_log.debug("ACK: syn not in msg.")
return
# Is this a reply to our SYN-ACK?
contract_id = self.contract_id(msg[u"syn_ack"][u"syn"])
if contract_id not in self.contracts:
_log.debug("ACK: contract not found.")
return
# Did I sign this?
if not self.is_valid_contract_sig(msg[u"syn_ack"]):
_log.debug("--------------")
_log.debug(msg)
_log.debug("--------------")
_log.debug(self.contracts)
_log.debug("--------------")
_log.debug("ACK: sig is invalid.")
return
# Is the syn valid?
if not self.is_valid_syn(msg[u"syn_ack"][u"syn"]):
_log.debug("ACK: syn is invalid.")
return
# Update handshake.
contract = self.contracts[contract_id]
self.handshake[contract_id] = {
"state": u"ACK",
"timestamp": time.time()
}
# Try make TCP con.
self.net.unl.connect(contract["src_unl"], {
"success":
success_wrapper(self, contract_id, contract["host_unl"])
},
force_master=0,
nonce=contract_id)
_log.debug("ACK")
def save_contract(self, contract):
# Record contract details.
contract_id = self.contract_id(contract)
self.contracts[contract_id] = contract
return contract_id
def send_msg(self, dict_obj, unl):
node_id = self.net.unl.deconstruct(unl)["node_id"]
msg = json.dumps(dict_obj, ensure_ascii=True)
self.net.dht_node.direct_message(node_id, msg)
def contract_id(self, contract):
if sys.version_info >= (3, 0, 0):
contract = str(contract).encode("ascii")
else:
contract = str(contract)
return hashlib.sha256(contract).hexdigest()
def sign_contract(self, contract):
if sys.version_info >= (3, 0, 0):
msg = str(contract).encode("ascii")
else:
msg = str(contract)
msg = binascii.hexlify(msg).decode("utf-8")
sig = self.wallet.sign_data(self.wif, msg)
if sys.version_info >= (3, 0, 0):
contract[u"signature"] = sig.decode("utf-8")
else:
contract[u"signature"] = unicode(sig)
return contract
def is_valid_contract_sig(self, contract):
sig = contract[u"signature"][:]
del contract[u"signature"]
if sys.version_info >= (3, 0, 0):
msg = str(contract).encode("ascii")
else:
msg = str(contract)
msg = binascii.hexlify(msg).decode("utf-8")
address = self.wallet.get_address(self.wif)
ret = self.wallet.verify_signature(address, sig, msg)
contract[u"signature"] = sig[:]
return ret
def simple_data_request(self, data_id, node_unl, direction):
file_size = 0
if direction == u"send":
action = u"upload"
else:
action = u"download"
return self.data_request(action, data_id, file_size, node_unl)
def data_request(self, action, data_id, file_size, node_unl):
"""
Action = put (upload), get (download.)
"""
_log.debug("In data request function")
# Who is hosting this data?
if action == "upload":
# We store this data.
direction = u"send"
host_unl = self.net.unl.value
assert (storage.manager.find(self.store_config, data_id)
is not None)
else:
# They store the data.
direction = u"receive"
host_unl = node_unl
if data_id in self.downloading:
raise Exception("Already trying to download this.")
# Encoding.
if sys.version_info >= (3, 0, 0):
if type(data_id) == bytes:
data_id = data_id.decode("utf-8")
if type(host_unl) == bytes:
host_unl = host_unl.decode("utf-8")
if type(node_unl) == bytes:
node_unl = node_unl.decode("utf-8")
else:
if type(data_id) == str:
data_id = unicode(data_id)
if type(host_unl) == str:
host_unl = unicode(host_unl)
if type(node_unl) == str:
node_unl = unicode(node_unl)
# Create contract.
contract = OrderedDict({
u"status": u"SYN",
u"direction": direction,
u"data_id": data_id,
u"file_size": file_size,
u"host_unl": host_unl,
u"dest_unl": node_unl,
u"src_unl": self.net.unl.value
})
# Sign contract.
contract = self.sign_contract(contract)
# Route contract.
contract_id = self.save_contract(contract)
self.send_msg(contract, node_unl)
_log.debug("Sending data request")
# Update handshake.
self.handshake[contract_id] = {
"state": "SYN",
"timestamp": time.time()
}
# For async code.
d = defer.Deferred()
self.defers[contract_id] = d
# Return defer for async code.
return d
def remove_file_from_storage(self, data_id):
storage.manager.remove(self.store_config, data_id)
def move_file_to_storage(self, path):
with open(path, "rb") as shard:
storage.manager.add(self.store_config, shard)
return {
"file_size": storage.shard.get_size(shard),
"data_id": storage.shard.get_id(shard)
}
def get_data_chunk(self, data_id, position, chunk_size=1048576):
path = storage.manager.find(self.store_config, data_id)
buf = b""
with open(path, "rb") as fp:
fp.seek(position, 0)
buf = fp.read(chunk_size)
return buf
def save_data_chunk(self, data_id, chunk):
_log.debug("Saving data chunk for " + str(data_id))
_log.debug("of size + " + str(len(chunk)))
assert (data_id in self.downloading)
# Find temp file path.
path = self.downloading[data_id]
_log.debug(path)
with open(path, "ab") as fp:
fp.write(chunk)
class TestFileHandshake(unittest.TestCase):
def setUp(self):
# Alice
self.alice_wallet = BtcTxStore(testnet=False, dryrun=True)
self.alice_wif = "L18vBLrz3A5QxJ6K4bUraQQZm6BAdjuAxU83e16y3x7eiiHTApHj"
self.alice_node_id = address_to_node_id(
self.alice_wallet.get_address(self.alice_wif)
)
self.alice_dht_node = pyp2p.dht_msg.DHT(
node_id=self.alice_node_id,
networking=0
)
self.alice_storage = tempfile.mkdtemp()
self.alice = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=0,
dht_node=self.alice_dht_node,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=self.alice_wif,
store_config={self.alice_storage: None}
)
# Bob
self.bob_wallet = BtcTxStore(testnet=False, dryrun=True)
self.bob_wif = "L3DBWWbuL3da2x7qAmVwBpiYKjhorJuAGobecCYQMCV7tZMAnDsr"
self.bob_node_id = address_to_node_id(
self.bob_wallet.get_address(self.bob_wif))
self.bob_dht_node = pyp2p.dht_msg.DHT(
node_id=self.bob_node_id,
networking=0
)
self.bob_storage = tempfile.mkdtemp()
self.bob = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=0,
dht_node=self.bob_dht_node,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=self.bob_wif,
store_config={self.bob_storage: None}
)
# Accept all transfers.
def accept_handler(contract_id, src_unl, data_id, file_size):
return 1
# Add accept handler.
self.alice.handlers["accept"].add(accept_handler)
self.bob.handlers["accept"].add(accept_handler)
# Link DHT nodes.
self.alice_dht_node.add_relay_link(self.bob_dht_node)
self.bob_dht_node.add_relay_link(self.alice_dht_node)
# Bypass sending messages for client.
def send_msg(dict_obj, unl):
print("Skipped sending message in test")
print(dict_obj)
print(unl)
# Install send msg hooks.
self.alice.send_msg = send_msg
self.bob.send_msg = send_msg
# Bypass sending relay messages for clients.
def relay_msg(node_id, msg):
print("Skipping relay message in test")
print(node_id)
print(msg)
# Install relay msg hooks.
if self.alice.net.dht_node is not None:
self.alice.net.dht_node.relay_message = relay_msg
if self.bob.net.dht_node is not None:
self.bob.net.dht_node.relay_message = relay_msg
# Bypass UNL.connect for clients.
def unl_connect(their_unl, events, force_master=1, hairpin=1,
nonce="0" * 64):
print("Skipping UNL.connect!")
print("Their unl = ")
print(their_unl)
print("Events = ")
print(events)
print("Force master = ")
print(force_master)
print("Hairpin = ")
print(hairpin)
print("Nonce = ")
print(nonce)
# Install UNL connect hooks.
self.alice.net.unl.connect = unl_connect
self.bob.net.unl.connect = unl_connect
# Record syn.
data_id = u"5feceb66ffc86f38d952786c6d696c79"
data_id += u"c2dbc239dd4e91b46729d73a27fb57e9"
self.syn = OrderedDict([
(u"status", u"SYN"),
(u"data_id", data_id),
(u"file_size", 100),
(u"host_unl", self.alice.net.unl.value),
(u"dest_unl", self.bob.net.unl.value),
(u"src_unl", self.alice.net.unl.value)
])
def tearDown(self):
self.alice.net.stop()
self.bob.net.stop()
def test_message_flow(self):
print("")
print("Testing message flow")
print("")
# Create file we're suppose to be uploading.
path = os.path.join(self.alice_storage, self.syn[u"data_id"])
if not os.path.exists(path):
with open(path, "w") as fp:
fp.write("0")
# Clear existing contracts.
self.clean_slate_all()
# Alice: build SYN.
contract_id = self.alice.simple_data_request(
data_id=self.syn[u"data_id"],
node_unl=self.bob.net.unl.value,
direction=u"send"
)
syn = self.alice.contracts[contract_id]
self.assertIsInstance(syn, OrderedDict)
print(self.alice.net.unl.value)
print(self.bob.net.unl.value)
print(syn)
# Bob: process SYN, build SYN-ACK.
syn_ack = process_syn(self.bob, syn)
self.assertIsInstance(syn_ack, OrderedDict)
# Alice: process SYN-ACK, build ACK.
ack = process_syn_ack(self.alice, syn_ack)
self.assertIsInstance(ack, OrderedDict)
# Bob: process ack.
fin = process_ack(self.bob, ack)
self.assertTrue(fin == 1)
print("")
print("Done testing message flow")
print("")
def clean_slate(self, client):
client.contracts = {}
client.cons = []
client.defers = {}
client.handshake = {}
client.con_info = {}
client.con_transfer = {}
client.downloading = {}
def clean_slate_all(self):
for client in [self.alice, self.bob]:
self.clean_slate(client)
def test_sign_syn(self):
print("")
print("Testing sign syn")
print("")
self.clean_slate_all()
syn = copy.deepcopy(self.syn)
signed_syn = self.alice.sign_contract(syn)
print(signed_syn)
print(self.alice.is_valid_contract_sig(signed_syn))
node_id = self.alice.net.dht_node.get_id()
print(node_id)
self.assertEqual(
self.alice.is_valid_contract_sig(signed_syn, node_id), 1
)
node_id = parse_node_id_from_unl(self.alice.net.unl.value)
self.assertEqual(
self.alice.is_valid_contract_sig(signed_syn, node_id), 1
)
print(node_id)
self.assertTrue(syn[u"src_unl"] == self.alice.net.unl.value)
print("Bob's perspective")
assert(self.bob.is_valid_contract_sig(signed_syn, node_id))
print("----")
print(signed_syn)
print("")
print("End sign syn")
print("")
def test_process_syn(self):
print("")
print("Testing process syn")
print("")
self.clean_slate_all()
syn = copy.deepcopy(self.syn)
# Create file we're suppose to be uploading.
path = os.path.join(self.alice_storage, syn[u"data_id"])
if not os.path.exists(path):
with open(path, "w") as fp:
fp.write("0")
# Test accept SYN with a handler.
def request_handler(contract_id, src_unl, data_id, file_size):
return 1
self.bob.handlers["accept"] = [request_handler]
syn = copy.deepcopy(self.syn)
self.assertIsInstance(process_syn(
self.bob, self.alice.sign_contract(syn), enable_accept_handlers=1
), OrderedDict)
del syn["signature"]
# Test reject SYN with a handler.
def request_handler(contract_id, src_unl, data_id, file_size):
return 0
self.bob.handlers["accept"] = [request_handler]
syn = copy.deepcopy(self.syn)
self.assertTrue(process_syn(
self.bob, self.alice.sign_contract(syn), enable_accept_handlers=1
) == -2)
del syn["signature"]
# Our UNL is incorrect.
syn = copy.deepcopy(self.syn)
syn[u"dest_unl"] = self.alice.net.unl.value
self.assertTrue(process_syn(
self.bob, self.alice.sign_contract(syn), enable_accept_handlers=0
) == -3)
syn[u"dest_unl"] = self.bob.net.unl.value
del syn["signature"]
# Their sig is invalid.
syn = copy.deepcopy(self.syn)
syn[u"signature"] = "x"
self.assertTrue(process_syn(
self.bob, syn, enable_accept_handlers=0
) == -4)
del syn["signature"]
# Handshake state is incorrect.
syn = copy.deepcopy(self.syn)
syn = self.alice.sign_contract(syn)
contract_id = self.bob.contract_id(syn)
self.bob.handshake[contract_id] = "SYN"
self.assertTrue(process_syn(
self.bob, syn, enable_accept_handlers=0
) == -5)
del self.bob.handshake[contract_id]
# This should pass.
self.assertIsInstance(process_syn(
self.bob, syn, enable_accept_handlers=0
), OrderedDict)
print("")
print("Ending process syn")
print("")
def test_valid_syn_ack(self):
print("")
print("Testing process syn-ack")
print("")
self.clean_slate_all()
syn = self.alice.sign_contract(copy.deepcopy(self.syn))
syn_ack = OrderedDict([(u'status', u'SYN-ACK'), (u'syn', syn)])
syn_ack = self.bob.sign_contract(syn_ack)
# Clear any old contracts that might exist.
self.alice.contracts = {}
# Create file we're suppose to be uploading.
path = os.path.join(self.alice_storage, syn_ack[u"syn"][u"data_id"])
if not os.path.exists(path):
with open(path, "w") as fp:
fp.write("0")
# Syn not in message.
syn_ack_2 = copy.deepcopy(syn_ack)
del syn_ack_2[u"syn"]
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -1)
# Invalid fields.
syn_ack_2 = copy.deepcopy(syn_ack)
syn_ack_2[u"xxx"] = "0"
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -2)
# Not a reply to something we sent.
syn_ack_2 = copy.deepcopy(syn_ack)
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -3)
# Save original SYN as a contract.
contract_id = self.alice.contract_id(syn_ack_2[u"syn"])
self.alice.contracts[contract_id] = syn_ack_2[u"syn"]
# Is SYN valid.
syn_ack_2 = copy.deepcopy(syn_ack)
syn_ack_2[u"syn"][u"file_size"] = "10"
contract_id = self.alice.contract_id(syn_ack_2[u"syn"])
self.alice.contracts[contract_id] = syn_ack_2[u"syn"]
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -4)
# Did we sign this?
syn_ack_2 = copy.deepcopy(syn_ack)
syn_ack_2[u"syn"][u"signature"] = "x"
contract_id = self.alice.contract_id(syn_ack_2[u"syn"])
self.alice.contracts[contract_id] = syn_ack_2[u"syn"]
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -5)
# Check their sig is valid.
syn_ack_2 = copy.deepcopy(syn_ack)
syn_ack_2[u"signature"] = "x"
contract_id = self.alice.contract_id(syn_ack_2[u"syn"])
self.alice.contracts[contract_id] = syn_ack_2[u"syn"]
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -6)
# Check handshake state is valid.
syn_ack_2 = copy.deepcopy(syn_ack)
self.alice.handshake = {}
ret = process_syn_ack(self.alice, syn_ack_2)
print("ERror 1")
print(ret)
self.assertTrue(ret == -7)
self.alice.handshake[contract_id] = {
u"state": u"ACK",
u"timestamp": time.time()
}
contract_id = self.alice.contract_id(syn_ack_2[u"syn"])
self.alice.contracts[contract_id] = syn_ack_2[u"syn"]
self.assertTrue(process_syn_ack(self.alice, syn_ack_2) == -8)
self.alice.handshake[contract_id] = {
u"state": u"SYN",
u"timestamp": time.time()
}
# This should pass.
syn_ack_2 = copy.deepcopy(syn_ack)
contract_id = self.alice.contract_id(syn_ack_2[u"syn"])
self.alice.contracts[contract_id] = syn_ack_2[u"syn"]
ret = process_syn_ack(self.alice, syn_ack_2)
print(ret)
self.assertIsInstance(ret, OrderedDict)
print("")
print("Ending process syn-ack")
print("")
def test_valid_ack(self):
print("")
print("Testing process ack")
print("")
self.clean_slate_all()
syn = self.alice.sign_contract(copy.deepcopy(self.syn))
syn_ack = OrderedDict([(u'status', u'SYN-ACK'), (u'syn', syn)])
syn_ack = self.bob.sign_contract(syn_ack)
ack = OrderedDict([(u'status', u'ACK'), (u'syn_ack', syn_ack)])
ack = self.alice.sign_contract(ack)
# SYN ack not in message.
ack_2 = copy.deepcopy(ack)
del ack_2[u"syn_ack"]
self.assertTrue(process_ack(self.bob, ack_2) == -1)
# Invalid length.
ack_2 = copy.deepcopy(ack)
ack_2["yy"] = 1
self.assertTrue(process_ack(self.bob, ack_2) == -2)
# Not a reply to our syn-ack.
ack_2 = copy.deepcopy(ack)
self.assertTrue(process_ack(self.bob, ack_2) == -3)
# Our sig is invalid.
ack_2 = copy.deepcopy(ack)
ack_2[u"syn_ack"][u"signature"] = "x"
contract_id = self.bob.contract_id(ack_2[u"syn_ack"][u"syn"])
self.bob.contracts[contract_id] = ack_2[u"syn_ack"][u"syn"]
self.assertTrue(process_ack(self.bob, ack_2) == -4)
# Contract ID not in handshakes.
ack_2 = copy.deepcopy(ack)
contract_id = self.bob.contract_id(ack_2[u"syn_ack"][u"syn"])
self.bob.contracts[contract_id] = ack_2[u"syn_ack"][u"syn"]
self.alice.handshake = {}
self.assertTrue(process_ack(self.bob, ack_2) == -5)
# Handshake state is invalid.
ack_2 = copy.deepcopy(ack)
contract_id = self.bob.contract_id(ack_2[u"syn_ack"][u"syn"])
self.bob.contracts[contract_id] = ack_2[u"syn_ack"][u"syn"]
self.bob.handshake[contract_id] = {
u"state": "SYN",
u"timestamp": time.time()
}
self.assertTrue(process_ack(self.bob, ack_2) == -6)
# This should pass.
ack_2 = copy.deepcopy(ack)
contract_id = self.bob.contract_id(ack_2[u"syn_ack"][u"syn"])
self.bob.contracts[contract_id] = ack_2[u"syn_ack"][u"syn"]
self.bob.handshake[contract_id] = {
u"state": "SYN-ACK",
u"timestamp": time.time()
}
ret = process_ack(self.bob, ack_2)
print(ret)
self.assertTrue(ret == 1)
print("")
print("Ending process ack")
print("")
def test_valid_rst(self):
print("")
print("Testing process rst")
print("")
self.clean_slate_all()
syn = self.alice.sign_contract(copy.deepcopy(self.syn))
# Rest contract state.
self.bob.contracts = {}
contract_id = self.alice.contract_id(syn)
rst = OrderedDict([
(u"status", u"RST"),
(u"contract_id", contract_id),
(u"src_unl", self.bob.net.unl.value)
])
# Contract ID not in message.
rst_2 = copy.deepcopy(rst)
del rst_2["contract_id"]
self.assertTrue(process_rst(self.alice, rst_2) == -1)
# SRC UNL not in message.
rst_2 = copy.deepcopy(rst)
del rst_2["src_unl"]
self.assertTrue(process_rst(self.alice, rst_2) == -2)
# Contract not found.
rst_2 = copy.deepcopy(rst)
self.assertTrue(process_rst(self.alice, rst_2) == -3)
# UNLs don't match for this contract.
self.alice.contracts[contract_id] = syn
rst_2 = copy.deepcopy(rst)
rst_2[u"src_unl"] = self.alice.net.unl.value
self.assertTrue(process_rst(self.alice, rst_2) == -4)
# Sig doesn't match for this contract.
rst_2 = copy.deepcopy(rst)
self.assertTrue(process_rst(self.alice, rst_2) == -5)
# This should pass.
rst_2 = copy.deepcopy(rst)
rst_2 = self.bob.sign_contract(rst_2)
self.assertTrue(process_rst(self.alice, rst_2) == 1)
# Setup callback.
def callback(ret):
global callbacks_work
callbacks_work = 1
# Check defer callbacks.
d = defer.Deferred()
self.alice.defers[contract_id] = d
d.addErrback(callback)
self.assertTrue(process_rst(self.alice, rst_2) == 1)
self.assertTrue(callbacks_work == 1)
print("")
print("Ending process rst")
print("")
def test_valid_syn(self):
print("")
print("Testing is_valid_syn")
print("")
self.clean_slate_all()
# Non existing fields.
syn = {}
self.assertTrue(is_valid_syn(self.alice, syn) == -1)
# Invalid number of fields.
syn = copy.deepcopy(self.syn)
syn["test"] = "test"
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)) == -2
)
del syn["test"]
del syn["signature"]
# The data ID is wrong.
syn["data_id"] = "x"
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)) == -3
)
syn["data_id"] = hashlib.sha256(b"0").hexdigest()
del syn["signature"]
# Syn is too big.
"""
syn[u"file_size"] = int("9" * (5242880 + 10))
self.assertTrue(is_valid_syn(
self.alice,
self.alice.sign_contract(syn)
) == -4)
syn[u"file_size"] = 1
"""
# Invalid UNLs.
syn["host_unl"] = "0"
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)) == -6
)
syn["host_unl"] = self.alice.net.unl.value
del syn["signature"]
# Invalid file size.
syn["file_size"] = str("0")
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)) == -7
)
syn["file_size"] = 20
del syn["signature"]
# We're the host and we don't have this file.
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)) == -8
)
del syn["signature"]
# We're not the host. We're downloading this.
# and we already have the file.
syn[u"host_unl"] = self.bob.net.unl.value
path = os.path.join(self.alice_storage, syn[u"data_id"])
if not os.path.exists(path):
with open(path, "w") as fp:
fp.write("0")
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)) == -9
)
del syn["signature"]
# We're not the host and we're already downloading this
os.remove(path)
self.alice.downloading[syn[u"data_id"]] = path
self.assertTrue(is_valid_syn(
self.alice,
self.alice.sign_contract(syn)
) == -10)
del self.alice.downloading[syn[u"data_id"]]
del syn["signature"]
# This should pass.
self.assertTrue(is_valid_syn(
self.alice, self.alice.sign_contract(syn)
) == 1)
print("")
print("Ending is_valid_syn")
print("")
from btctxstore import BtcTxStore __author__ = 'karatel' test_btctx_api = BtcTxStore(testnet=True, dryrun=True) test_owner_wif = test_btctx_api.create_key() test_owner_address = test_btctx_api.get_address(test_owner_wif) test_other_wfi = test_btctx_api.create_key() test_other_address = test_btctx_api.get_address(test_other_wfi)
class Client(object):
def __init__(self, url=common.DEFAULT_URL, debug=False, quiet=False,
use_folder_tree=False, max_size=common.DEFAULT_MAX_SIZE,
min_free_size=common.DEFAULT_MIN_FREE_SIZE,
store_path=common.DEFAULT_STORE_PATH,
config_path=common.DEFAULT_CONFIG_PATH,
connection_retry_limit=common.DEFAULT_CONNECTION_RETRY_LIMIT,
connection_retry_delay=common.DEFAULT_CONNECTION_RETRY_DELAY):
debug = deserialize.flag(debug)
quiet = deserialize.flag(quiet)
self.url = deserialize.url(url)
self.use_folder_tree = deserialize.flag(use_folder_tree)
self.max_size = deserialize.byte_count(max_size)
self.min_free_size = deserialize.byte_count(min_free_size)
self.messenger = None # lazy
self.btctxstore = BtcTxStore()
self.retry_limit = deserialize.positive_integer(connection_retry_limit)
self.retry_delay = deserialize.positive_integer(connection_retry_delay)
# paths
self.cfg_path = os.path.realpath(config_path)
storjnode.util.ensure_path_exists(os.path.dirname(self.cfg_path))
self.store_path = os.path.realpath(store_path)
storjnode.util.ensure_path_exists(self.store_path)
# check for vfat partions
try:
fstype = storjnode.util.get_fs_type(self.store_path)
# FileNotFoundError: [Errno 2] No such file or directory: '/etc/mtab'
# psutil: https://code.google.com/p/psutil/issues/detail?id=434
except EnvironmentError as e:
logger.warning(e)
fstype = None
if fstype == "vfat":
logger.info("Detected vfat partition, using folder tree.")
self.use_folder_tree = True
if fstype is None:
msg = "Couldn't detected partition type for '{0}'"
logger.warning(msg.format(self.store_path))
self.cfg = storjnode.config.get(self.btctxstore, self.cfg_path)
@staticmethod
def version():
print(__version__)
return __version__
def _init_messenger(self):
"""Make sure messenger exists."""
if self.messenger is None:
wif = self.btctxstore.get_key(self.cfg["wallet"])
self.messenger = messaging.Messaging(self.url, wif,
self.retry_limit,
self.retry_delay)
def register(self):
"""Attempt to register the config address."""
self._init_messenger()
payout_address = self.cfg["payout_address"]
self.messenger.register(payout_address)
logger.info("Registered on server '{0}'.".format(self.url))
return True
def config(self, set_wallet=None, set_payout_address=None):
"""
Set and then show the config settings.
:param set_wallet: Set the HWIF for registration/auth address.
:param set_payout_address: Set the payout address.
:return: Configuation object.
"""
if((set_payout_address is not None) and
(not self.btctxstore.validate_address(set_payout_address))):
raise exceptions.InvalidAddress(set_payout_address)
if((set_wallet is not None) and
(not self.btctxstore.validate_wallet(set_wallet))):
raise exceptions.InvalidHWIF(set_wallet)
self._init_messenger()
config_updated = False
# update payout address if requested
if set_payout_address:
self.cfg["payout_address"] = set_payout_address
config_updated = True
# update wallet if requested
if set_wallet:
self.cfg["wallet"] = set_wallet
config_updated = True
# save config if updated
if config_updated:
storjnode.config.save(self.btctxstore, self.cfg_path, self.cfg)
# display config
print(SHOW_CONFIG_TEMPLATE.format(
self.messenger.get_nodeid(),
self.cfg["payout_address"]
))
return self.cfg
def ping(self):
"""Attempt one keep-alive with the server."""
self._init_messenger()
msg = "Pinging server '{0}' at {1:%Y-%m-%d %H:%M:%S}."
logger.info(msg.format(self.messenger.server_url(), datetime.now()))
self.messenger.ping()
return True
def poll(self, delay=common.DEFAULT_DELAY, limit=None):
"""Attempt continuous keep-alive with the server.
:param delay: Delay in seconds per ping of the server.
:param limit: Number of seconds in the future to stop polling.
:return: True, if limit is reached. None, if otherwise.
"""
delay = deserialize.positive_integer(delay)
stop_time = None
if limit is not None:
stop_time = datetime.now() + timedelta(seconds=int(limit))
while True: # ping the server every X seconds
self.ping()
if stop_time and datetime.now() >= stop_time:
return True
time.sleep(int(delay))
def freespace(self):
freespace = psutil.disk_usage(self.store_path).free
print(freespace)
return freespace
def build(self, workers=1, cleanup=False, rebuild=False, repair=False,
set_height_interval=common.DEFAULT_SET_HEIGHT_INTERVAL):
"""Generate test files deterministically based on address.
:param workers: Number of Number of threadpool workers.
:param cleanup: Remove files in shard directory.
:param rebuild: Re-generate any file shards.
:param set_height_interval: Number of shards to generate before
notifying the server.
"""
workers = deserialize.positive_nonzero_integer(workers)
set_height_interval = deserialize.positive_nonzero_integer(
set_height_interval
)
cleanup = deserialize.flag(cleanup)
rebuild = deserialize.flag(rebuild)
repair = deserialize.flag(repair)
self._init_messenger()
logger.info("Starting build")
def _on_generate_shard(cur_height, last):
"""
Because URL requests are slow, only update the server when we are
at the first height, at some height_interval, or the last height.
:param cur_height: Current height in the building process.
"""
first = cur_height == 1
set_height = (cur_height % int(set_height_interval)) == 0
if first or set_height or last:
self.messenger.height(cur_height)
logger.info("Current height at {0}.".format(cur_height))
# Initialize builder and generate/re-generate shards
bldr = builder.Builder(address=self.cfg["payout_address"],
shard_size=common.SHARD_SIZE,
max_size=self.max_size,
min_free_size=self.min_free_size,
on_generate_shard=_on_generate_shard,
use_folder_tree=self.use_folder_tree)
generated = bldr.build(self.store_path, workers=workers,
cleanup=cleanup, rebuild=rebuild, repair=repair)
logger.info("Build finished")
return generated
def audit(self, delay=common.DEFAULT_AUDIT_DELAY, limit=None):
self._init_messenger()
# Initialize builder and audit shards
bldr = builder.Builder(address=self.cfg["payout_address"],
shard_size=common.SHARD_SIZE,
max_size=self.max_size,
min_free_size=self.min_free_size,
use_folder_tree=self.use_folder_tree)
delay = deserialize.positive_integer(delay)
stop_time = None
if limit is not None:
stop_time = datetime.now() + timedelta(seconds=int(limit))
btc_index = 0
while True:
btc_block = bldr.btc_last_confirmed_block(
min_confirmations=common.DEFAULT_MIN_CONFIRMATIONS
)
if btc_block['block_no'] != btc_index:
btc_hash = btc_block['blockhash']
btc_index = btc_block['block_no']
logger.debug("Using bitcoin block {0} hash {1}.".format(
btc_index, btc_hash))
wif = self.btctxstore.get_key(self.cfg["wallet"])
address = self.btctxstore.get_address(wif)
response_data = address + btc_hash + str(bldr.audit(
self.store_path,
btc_block['block_no'],
btc_block['blockhash']))
response = hashlib.sha256(
response_data.encode('utf-8')
).hexdigest()
# New Dataserv Server version is needed
self.messenger.audit(btc_block['block_no'], response)
else:
msg = "Bitcoin block {0} already used. Waiting for new block."
logger.debug(msg.format(btc_index))
if stop_time and datetime.now() >= stop_time:
return True
time.sleep(int(delay))
def farm(self, workers=1, cleanup=False, rebuild=False, repair=False,
set_height_interval=common.DEFAULT_SET_HEIGHT_INTERVAL,
delay=common.DEFAULT_DELAY, limit=None):
""" Fully automatic client for users wishing a simple turnkey solution.
This will run all functions automatically with the most sane defaults
and as little user interface as possible.
:param workers: Number of Number of threadpool workers.
:param cleanup: Remove files in shard directory.
:param rebuild: Re-generate any file shards.
:param set_height_interval: Number of shards to generate before
notifying the server.
:param delay: Delay in seconds per ping of the server.
:param limit: Number of seconds in the future to stop polling.
"""
workers = deserialize.positive_nonzero_integer(workers)
set_height_interval = deserialize.positive_nonzero_integer(
set_height_interval
)
cleanup = deserialize.flag(cleanup)
rebuild = deserialize.flag(rebuild)
repair = deserialize.flag(repair)
# farmer never gives up
self._init_messenger()
self.messenger.retry_limit = 99999999999999999999999999999999999999
try:
self.register()
except exceptions.AddressAlreadyRegistered:
pass # already registered ...
self.set_bandwidth()
self.build(workers=workers, cleanup=cleanup, rebuild=rebuild,
repair=repair, set_height_interval=set_height_interval)
self.poll(delay=delay, limit=limit)
return True
def set_bandwidth(self):
results = speedtest()
self.messenger.set_bandwidth(results["upload"],
results["download"])
class FileTransfer:
def __init__(self, net, wif=None, store_config=None, handlers=None):
# Accept direct connections.
self.net = net
# Returned by callbacks.
self.success_value = ("127.0.0.1", 7777)
# Used for signing messages.
self.wallet = BtcTxStore(testnet=True, dryrun=True)
self.wif = wif or self.wallet.create_key()
# Where will the data be stored?
self.store_config = store_config
assert(len(list(store_config)))
# Handlers for certain events.
self.handlers = handlers
# Start networking.
if not self.net.is_net_started:
self.net.start()
# Dict of data requests.
self.contracts = {}
# Dict of defers for contracts.
self.defers = {}
# Three-way handshake status for contracts.
self.handshake = {}
# All contracts associated with this connection.
self.con_info = {}
# File transfer currently active on connection.
self.con_transfer = {}
# List of active downloads.
# (Never try to download multiple copies of the same thing at once.)
self.downloading = {}
def get_their_unl(self, contract):
if self.net.unl == pyp2p.unl.UNL(value=contract["dest_unl"]):
their_unl = contract["src_unl"]
else:
their_unl = contract["dest_unl"]
return their_unl
def is_queued(self, con=None):
if con is not None:
if con not in self.con_info:
return 0
if con is None:
con_list = list(self.con_info)
else:
con_list = [con]
for con in con_list:
for contract_id in list(self.con_info[con]):
con_info = self.con_info[con][contract_id]
if con_info["remaining"]:
return 1
return 0
def cleanup_transfers(self, con):
# Close con - there's nothing left to download.
if not self.is_queued(con):
# Cleanup con transfers.
if con in self.con_transfer:
del self.con_transfer[con]
# Cleanup con_info.
if con in self.con_info:
del self.con_info[con]
# Todo: cleanup contract + handshake state.
def queue_next_transfer(self, con):
_log.debug("Queing next transfer")
for contract_id in list(self.con_info[con]):
con_info = self.con_info[con][contract_id]
if con_info["remaining"]:
self.con_transfer[con] = contract_id
con.send(contract_id, send_all=1)
return
# Mark end of transfers.
self.con_transfer[con] = u"0" * 64
def is_valid_syn(self, msg):
# List of expected fields.
syn_schema = (
u"status",
u"direction",
u"data_id",
u"file_size",
u"host_unl",
u"dest_unl",
u"src_unl",
u"signature"
)
# Check all fields exist.
if not all(key in msg for key in syn_schema):
_log.debug("Missing required key.")
return 0
# Check SYN size.
if len(msg) > 5242880: # 5 MB.
_log.debug("SYN is too big")
return 0
# Check direction is valid.
direction_tuple = (u"send", u"receive")
if msg[u"direction"] not in direction_tuple:
_log.debug("Missing required direction tuple.")
return 0
# Check the UNLs are valid.
unl_tuple = (u"host_unl", u"dest_unl", u"src_unl")
for unl_key in unl_tuple:
if not pyp2p.unl.is_valid_unl(msg[unl_key]):
_log.debug("Invalid UNL for " + unl_key)
_log.debug(msg[unl_key])
return 0
# Check file size.
file_size_type = type(msg[u"file_size"])
if sys.version_info >= (3, 0, 0):
expr = file_size_type != int
else:
expr = file_size_type != int and file_size_type != long
if expr:
_log.debug("File size validation failed")
_log.debug(type(msg[u"file_size"]))
return 0
# Are we the host?
if self.net.unl == pyp2p.unl.UNL(value=msg[u"host_unl"]):
# Then check we have this file.
path = storage.manager.find(self.store_config, msg[u"data_id"])
if path is None:
_log.debug("Failed to find file we're uploading")
return 0
else:
# Do we already have this file?
path = storage.manager.find(self.store_config, msg[u"data_id"])
if path is not None:
_log.debug("Attempting to download file we already have")
return 0
# Are we already trying to download this?
if msg[u"data_id"] in self.downloading:
_log.debug("We're already trying to download this")
return 0
return 1
def protocol(self, msg):
msg = json.loads(msg, object_pairs_hook=OrderedDict)
# Associate TCP con with contract.
def success_wrapper(self, contract_id, host_unl):
def success(con):
with mutex:
_log.debug("IN SUCCESS CALLBACK")
_log.debug("Success() contract_id = " + str(contract_id))
# Associate TCP con with contract.
contract = self.contracts[contract_id]
file_size = contract["file_size"]
# Store con association.
if con not in self.con_info:
self.con_info[con] = {}
# Associate contract with con.
if contract_id not in self.con_info[con]:
self.con_info[con][contract_id] = {
"contract_id": contract_id,
"remaining": 350, # Tree fiddy.
"file_size": file_size,
"file_size_buf": b""
}
# Record download state.
data_id = contract["data_id"]
if self.net.unl != pyp2p.unl.UNL(value=host_unl):
_log.debug("Success: download")
fp, self.downloading[data_id] = tempfile.mkstemp()
else:
# Set initial upload for this con.
_log.debug("Success: upload")
# Queue first transfer.
their_unl = self.get_their_unl(contract)
is_master = self.net.unl.is_master(their_unl)
_log.debug("Is master = " + str(is_master))
if con not in self.con_transfer:
if is_master:
# A transfer to queue processing.
self.queue_next_transfer(con)
else:
# A transfer to receive (unknown.)
self.con_transfer[con] = u""
else:
if self.con_transfer[con] == u"0" * 64:
if is_master:
self.queue_next_transfer(con)
else:
self.con_transfer[con] = u""
return success
# Sanity checking.
if u"status" not in msg:
return
# Accept data request.
if msg[u"status"] == u"SYN":
# Check syn is valid.
if not self.is_valid_syn(msg):
_log.debug("SYN: invalid syn.")
return
# Save contract.
contract_id = self.contract_id(msg)
self.save_contract(msg)
self.handshake[contract_id] = {
"state": u"SYN-ACK",
"timestamp": time.time()
}
# Create reply.
reply = OrderedDict({
u"status": u"SYN-ACK",
u"syn": msg,
})
# Sign reply.
reply = self.sign_contract(reply)
# Save reply.
self.send_msg(reply, msg[u"src_unl"])
_log.debug("SYN")
# Confirm accept and make connection if needed.
if msg[u"status"] == u"SYN-ACK":
# Valid syn-ack?
if u"syn" not in msg:
_log.debug("SYN-ACK: syn not in msg.")
return
# Is this a reply to our SYN?
contract_id = self.contract_id(msg[u"syn"])
if contract_id not in self.contracts:
_log.debug("--------------")
_log.debug(msg)
_log.debug("--------------")
_log.debug(self.contracts)
_log.debug("--------------")
_log.debug("SYN-ACK: contract not found.")
return
# Check syn is valid.
if not self.is_valid_syn(msg[u"syn"]):
_log.debug("SYN-ACK: invalid syn.")
return
# Did I sign this?
if not self.is_valid_contract_sig(msg[u"syn"]):
_log.debug("SYN-ACK: sig is invalid.")
return
# Update handshake.
contract = self.contracts[contract_id]
self.handshake[contract_id] = {
"state": u"ACK",
"timestamp": time.time()
}
# Create reply contract.
reply = OrderedDict({
u"status": u"ACK",
u"syn_ack": msg
})
# Sign reply.
reply = self.sign_contract(reply)
# Try make TCP con.
self.net.unl.connect(
contract["dest_unl"],
{
"success": success_wrapper(
self,
contract_id,
contract["host_unl"]
)
},
force_master=0,
nonce=contract_id
)
# Send reply.
self.send_msg(reply, msg[u"syn"][u"dest_unl"])
_log.debug("SYN-ACK")
if msg[u"status"] == u"ACK":
# Valid ack.
if u"syn_ack" not in msg:
_log.debug("ACK: syn_ack not in msg.")
return
if u"syn" not in msg[u"syn_ack"]:
_log.debug("ACK: syn not in msg.")
return
# Is this a reply to our SYN-ACK?
contract_id = self.contract_id(msg[u"syn_ack"][u"syn"])
if contract_id not in self.contracts:
_log.debug("ACK: contract not found.")
return
# Did I sign this?
if not self.is_valid_contract_sig(msg[u"syn_ack"]):
_log.debug("--------------")
_log.debug(msg)
_log.debug("--------------")
_log.debug(self.contracts)
_log.debug("--------------")
_log.debug("ACK: sig is invalid.")
return
# Is the syn valid?
if not self.is_valid_syn(msg[u"syn_ack"][u"syn"]):
_log.debug("ACK: syn is invalid.")
return
# Update handshake.
contract = self.contracts[contract_id]
self.handshake[contract_id] = {
"state": u"ACK",
"timestamp": time.time()
}
# Try make TCP con.
self.net.unl.connect(
contract["src_unl"],
{
"success": success_wrapper(
self,
contract_id,
contract["host_unl"]
)
},
force_master=0,
nonce=contract_id
)
_log.debug("ACK")
def save_contract(self, contract):
# Record contract details.
contract_id = self.contract_id(contract)
self.contracts[contract_id] = contract
return contract_id
def send_msg(self, dict_obj, unl):
node_id = self.net.unl.deconstruct(unl)["node_id"]
msg = json.dumps(dict_obj, ensure_ascii=True)
self.net.dht_node.direct_message(
node_id,
msg
)
def contract_id(self, contract):
if sys.version_info >= (3, 0, 0):
contract = str(contract).encode("ascii")
else:
contract = str(contract)
return hashlib.sha256(contract).hexdigest()
def sign_contract(self, contract):
if sys.version_info >= (3, 0, 0):
msg = str(contract).encode("ascii")
else:
msg = str(contract)
msg = binascii.hexlify(msg).decode("utf-8")
sig = self.wallet.sign_data(self.wif, msg)
if sys.version_info >= (3, 0, 0):
contract[u"signature"] = sig.decode("utf-8")
else:
contract[u"signature"] = unicode(sig)
return contract
def is_valid_contract_sig(self, contract):
sig = contract[u"signature"][:]
del contract[u"signature"]
if sys.version_info >= (3, 0, 0):
msg = str(contract).encode("ascii")
else:
msg = str(contract)
msg = binascii.hexlify(msg).decode("utf-8")
address = self.wallet.get_address(self.wif)
ret = self.wallet.verify_signature(address, sig, msg)
contract[u"signature"] = sig[:]
return ret
def simple_data_request(self, data_id, node_unl, direction):
file_size = 0
if direction == u"send":
action = u"upload"
else:
action = u"download"
return self.data_request(action, data_id, file_size, node_unl)
def data_request(self, action, data_id, file_size, node_unl):
"""
Action = put (upload), get (download.)
"""
_log.debug("In data request function")
# Who is hosting this data?
if action == "upload":
# We store this data.
direction = u"send"
host_unl = self.net.unl.value
assert(storage.manager.find(self.store_config, data_id) is not None)
else:
# They store the data.
direction = u"receive"
host_unl = node_unl
if data_id in self.downloading:
raise Exception("Already trying to download this.")
# Encoding.
if sys.version_info >= (3, 0, 0):
if type(data_id) == bytes:
data_id = data_id.decode("utf-8")
if type(host_unl) == bytes:
host_unl = host_unl.decode("utf-8")
if type(node_unl) == bytes:
node_unl = node_unl.decode("utf-8")
else:
if type(data_id) == str:
data_id = unicode(data_id)
if type(host_unl) == str:
host_unl = unicode(host_unl)
if type(node_unl) == str:
node_unl = unicode(node_unl)
# Create contract.
contract = OrderedDict({
u"status": u"SYN",
u"direction": direction,
u"data_id": data_id,
u"file_size": file_size,
u"host_unl": host_unl,
u"dest_unl": node_unl,
u"src_unl": self.net.unl.value
})
# Sign contract.
contract = self.sign_contract(contract)
# Route contract.
contract_id = self.save_contract(contract)
self.send_msg(contract, node_unl)
_log.debug("Sending data request")
# Update handshake.
self.handshake[contract_id] = {
"state": "SYN",
"timestamp": time.time()
}
# For async code.
d = defer.Deferred()
self.defers[contract_id] = d
# Return defer for async code.
return d
def remove_file_from_storage(self, data_id):
storage.manager.remove(self.store_config, data_id)
def move_file_to_storage(self, path):
with open(path, "rb") as shard:
storage.manager.add(self.store_config, shard)
return {
"file_size": storage.shard.get_size(shard),
"data_id": storage.shard.get_id(shard)
}
def get_data_chunk(self, data_id, position, chunk_size=1048576):
path = storage.manager.find(self.store_config, data_id)
buf = b""
with open(path, "rb") as fp:
fp.seek(position, 0)
buf = fp.read(chunk_size)
return buf
def save_data_chunk(self, data_id, chunk):
_log.debug("Saving data chunk for " + str(data_id))
_log.debug("of size + " + str(len(chunk)))
assert(data_id in self.downloading)
# Find temp file path.
path = self.downloading[data_id]
_log.debug(path)
with open(path, "ab") as fp:
fp.write(chunk)
def test_queued():
from crochet import setup
setup()
# Alice sample node.
alice_wallet = BtcTxStore(testnet=False, dryrun=True)
alice_wif = alice_wallet.create_key()
alice_node_id = address_to_node_id(alice_wallet.get_address(alice_wif))
alice_dht = pyp2p.dht_msg.DHT(
node_id=alice_node_id,
networking=0
)
alice = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63400,
dht_node=alice_dht,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=alice_wif,
store_config={tempfile.mkdtemp(): None},
)
# Bob sample node.
bob_wallet = BtcTxStore(testnet=False, dryrun=True)
bob_wif = bob_wallet.create_key()
bob_node_id = address_to_node_id(bob_wallet.get_address(bob_wif))
bob_dht = pyp2p.dht_msg.DHT(
node_id=bob_node_id,
networking=0
)
bob = FileTransfer(
pyp2p.net.Net(
net_type="direct",
node_type="passive",
nat_type="preserving",
passive_port=63401,
dht_node=bob_dht,
wan_ip="8.8.8.8",
debug=1
),
BandwidthLimit(),
wif=bob_wif,
store_config={tempfile.mkdtemp(): None}
)
# Simulate Alice + Bob "connecting"
alice_dht.add_relay_link(bob_dht)
bob_dht.add_relay_link(alice_dht)
# Accept all transfers.
def accept_handler(contract_id, src_unl, data_id, file_size):
return 1
# Add accept handler.
alice.handlers["accept"].add(accept_handler)
bob.handlers["accept"].add(accept_handler)
# Create file we're suppose to be uploading.
data_id = ("5feceb66ffc86f38d952786c6d696c"
"79c2dbc239dd4e91b46729d73a27fb57e9")
path = os.path.join(list(alice.store_config)[0], data_id)
if not os.path.exists(path):
with open(path, "w") as fp:
fp.write("0")
# Alice wants to upload data to Bob.
upload_contract_id = alice.data_request(
"download",
data_id,
0,
bob.net.unl.value
)
# Delete source file.
def callback_builder(path, alice, bob, data_id):
def callback(client, contract_id, con):
print("Upload succeeded")
print("Removing content and downloading back")
os.remove(path)
# Fix transfers.
bob.handlers["complete"] = []
# Synchronize cons and check con.unl.
time.sleep(1)
clients = {"alice": alice, "bob": bob}
for client in list({"alice": alice, "bob": bob}):
print()
print(client)
clients[client].net.synchronize()
nodes_out = clients[client].net.outbound
nodes_in = clients[client].net.inbound
for node in nodes_out + nodes_in:
print(node["con"].unl)
print(clients[client].cons)
# Queued transfer:
download_contract_id = alice.data_request(
"upload",
data_id,
0,
bob.net.unl.value
)
print("Download contract ID =")
print(download_contract_id)
# Indicate Bob's download succeeded.
def alice_callback(val):
print("Download succeeded")
global queue_succeeded
queue_succeeded = 1
def alice_errback(val):
print("Download failed! Error:")
print(val)
# Hook upload from bob.
d = alice.defers[download_contract_id]
d.addCallback(alice_callback)
d.addErrback(alice_errback)
return callback
# Register callback for bob (when he's downloaded the data.)
bob.handlers["complete"] = [
callback_builder(path, alice, bob, data_id)
]
# d = alice.defers[upload_contract_id]
# d.addCallback(callback_builder(path, alice, bob, data_id))
# Main event loop.
timeout = time.time() + 40
while not queue_succeeded and time.time() < timeout:
for client in [alice, bob]:
if client == alice:
_log.debug("Alice")
else:
_log.debug("Bob")
process_transfers(client)
time.sleep(1)
if not queue_succeeded:
print("\a")
for client in [alice, bob]:
client.net.stop()
assert(queue_succeeded == 1)
class TestConfig(unittest.TestCase):
def setUp(self):
self.btctxstore = BtcTxStore()
def test_roundtrip_unencrypted(self):
path = tempfile.mktemp()
saved_data = config.create(self.btctxstore, path)
loaded_data = config.get(self.btctxstore, path)
self.assertEqual(saved_data, loaded_data)
os.remove(path)
def test_save_overwrites(self):
path = tempfile.mktemp()
# create config
created_data = config.create(self.btctxstore, path)
# update config
updated_data = copy.deepcopy(created_data)
updated_data["payout_address"] = "1A8WqiJDh3tGVeEefbMN5BVDYxx2XSoWgG"
config.save(self.btctxstore, path, updated_data)
# confirm overwriten
loaded_data = config.get(self.btctxstore, path)
self.assertEqual(updated_data, loaded_data)
os.remove(path)
def test_password_validation(self):
pass # TODO implement
def test_validation(self):
wallet = self.btctxstore.create_wallet()
key = self.btctxstore.get_key(wallet)
address = self.btctxstore.get_address(key)
# must be a dict
def callback():
config.validate(self.btctxstore, None)
self.assertRaises(exceptions.InvalidConfig, callback)
# must have the correct version
def callback():
config.validate(self.btctxstore, {
"payout_address": address,
"wallet": wallet,
})
self.assertRaises(exceptions.InvalidConfig, callback)
# must have a valid payout address
def callback():
config.validate(self.btctxstore, {
"version": __version__,
"wallet": wallet,
})
self.assertRaises(exceptions.InvalidConfig, callback)
# must have a valid wallet
def callback():
config.validate(self.btctxstore, {
"version": __version__,
"payout_address": address,
})
self.assertRaises(exceptions.InvalidConfig, callback)
# valid config
self.assertTrue(
config.validate(
self.btctxstore, {
"version": __version__,
"payout_address": address,
"wallet": wallet,
}))
def test_migrate(self):
path = tempfile.mktemp()
# initial unmigrated 2.0.0 config
cfg = {
"version": "2.0.0",
"master_secret": "test_master_secret",
"payout_address": "1A8WqiJDh3tGVeEefbMN5BVDYxx2XSoWgG",
}
# test its invalid with current build
def callback():
config.validate(self.btctxstore, cfg)
self.assertRaises(exceptions.InvalidConfig, callback)
# migrate
cfg = config.migrate(self.btctxstore, path, cfg)
# test its now valid
self.assertTrue(config.validate(self.btctxstore, cfg))
