def setUp(self, **kwargs):
yield super(TestDatabase, self).setUp()
path = os.path.join(self.getStateDir(), DATABASE_DIRECTORY)
if not os.path.exists(path):
os.makedirs(path)
self.db = MultiChainDB(self.getStateDir())
self.block1 = TestBlock()
self.block2 = TestBlock()
def __init__(self, *args, **kwargs):
super(MultiChainCommunity, self).__init__(*args, **kwargs)
self.logger = logging.getLogger(self.__class__.__name__)
self.notifier = None
self.persistence = MultiChainDB(self.dispersy.working_directory)
# We store the bytes send and received in the tunnel community in a dictionary.
# The key is the public key of the peer being interacted with, the value a tuple of the up and down bytes
# This data is not used to create outgoing requests, but _only_ to verify incoming requests
self.pending_bytes = dict()
self.logger.debug(
"The multichain community started with Public Key: %s",
self.my_member.public_key.encode("hex"))
def __init__(self, *args, **kwargs):
super(MultiChainCommunity, self).__init__(*args, **kwargs)
self.logger = logging.getLogger(self.__class__.__name__)
self.notifier = None
self._private_key = self.my_member.private_key
self._public_key = self.my_member.public_key
self.persistence = MultiChainDB(self.dispersy,
self.dispersy.working_directory)
self.logger.debug(
"The multichain community started with Public Key: %s",
base64.encodestring(self._public_key))
# No response is expected yet.
self.expected_response = None
def set_db_version(self, version):
self.db.executescript(
u"UPDATE option SET value = '%d' WHERE key = 'database_version';" %
version)
self.db.close(commit=True)
self.db = MultiChainDB(self.getStateDir())
class TestDatabase(MultiChainTestCase):
"""
Tests the Database for MultiChain community.
"""
def __init__(self, *args, **kwargs):
super(TestDatabase, self).__init__(*args, **kwargs)
@blocking_call_on_reactor_thread
@inlineCallbacks
def setUp(self, **kwargs):
yield super(TestDatabase, self).setUp()
path = os.path.join(self.getStateDir(), DATABASE_DIRECTORY)
if not os.path.exists(path):
os.makedirs(path)
self.db = MultiChainDB(self.getStateDir())
self.block1 = TestBlock()
self.block2 = TestBlock()
@blocking_call_on_reactor_thread
def test_add_block(self):
# Act
self.db.add_block(self.block1)
# Assert
result = self.db.get_latest(self.block1.public_key)
self.assertEqual_block(self.block1, result)
@blocking_call_on_reactor_thread
def test_get_num_interactors(self):
"""
Test whether the right number of interactors is returned
"""
self.block2 = TestBlock(previous=self.block1)
self.db.add_block(self.block1)
self.db.add_block(self.block2)
self.assertEqual(
(2, 2), self.db.get_num_unique_interactors(self.block1.public_key))
@blocking_call_on_reactor_thread
def test_add_two_blocks(self):
# Act
self.db.add_block(self.block1)
self.db.add_block(self.block2)
# Assert
result = self.db.get_latest(self.block2.public_key)
self.assertEqual_block(self.block2, result)
@blocking_call_on_reactor_thread
def test_get_block_non_existing(self):
# Act
result = self.db.get_latest(self.block1.public_key)
# Assert
self.assertEqual(None, result)
@blocking_call_on_reactor_thread
def test_contains_block_id_positive(self):
# Act
self.db.add_block(self.block1)
# Assert
self.assertTrue(self.db.contains(self.block1))
@blocking_call_on_reactor_thread
def test_contains_block_id_negative(self):
# Act & Assert
self.assertFalse(self.db.contains(self.block1))
@blocking_call_on_reactor_thread
def test_get_linked_forward(self):
# Arrange
self.block2 = TestBlock.create(self.db,
self.block2.public_key,
link=self.block1)
self.db.add_block(self.block1)
self.db.add_block(self.block2)
# Act
result = self.db.get_linked(self.block1)
# Assert
self.assertEqual_block(self.block2, result)
@blocking_call_on_reactor_thread
def test_get_linked_backwards(self):
# Arrange
self.block2 = TestBlock.create(self.db,
self.block2.public_key,
link=self.block1)
self.db.add_block(self.block1)
self.db.add_block(self.block2)
# Act
result = self.db.get_linked(self.block2)
# Assert
self.assertEqual_block(self.block1, result)
@blocking_call_on_reactor_thread
def test_get_block_after(self):
# Arrange
self.block2.public_key = self.block1.public_key
self.block2.sequence_number = self.block1.sequence_number + 1
block3 = TestBlock()
block3.public_key = self.block2.public_key
block3.sequence_number = self.block2.sequence_number + 10
self.db.add_block(self.block1)
self.db.add_block(self.block2)
self.db.add_block(block3)
# Act
result = self.db.get_block_after(self.block2)
# Assert
self.assertEqual_block(block3, result)
@blocking_call_on_reactor_thread
def test_get_block_before(self):
# Arrange
self.block2.public_key = self.block1.public_key
self.block2.sequence_number = self.block1.sequence_number + 1
block3 = TestBlock()
block3.public_key = self.block2.public_key
block3.sequence_number = self.block2.sequence_number + 10
self.db.add_block(self.block1)
self.db.add_block(self.block2)
self.db.add_block(block3)
# Act
result = self.db.get_block_before(self.block2)
# Assert
self.assertEqual_block(self.block1, result)
@blocking_call_on_reactor_thread
def test_save_large_upload_download_block(self):
"""
Test if the block can save very large numbers.
"""
# Arrange
self.block1.total_up = long(pow(2, 62))
self.block1.total_down = long(pow(2, 62))
# Act
self.db.add_block(self.block1)
# Assert
result = self.db.get_latest(self.block1.public_key)
self.assertEqual_block(self.block1, result)
@blocking_call_on_reactor_thread
def test_get_insert_time(self):
# Arrange
# Upon adding the block to the database, the timestamp will get added.
self.db.add_block(self.block1)
# Act
# Retrieving the block from the database will result in a block with a timestamp
result = self.db.get_latest(self.block1.public_key)
insert_time = datetime.datetime.strptime(result.insert_time,
"%Y-%m-%d %H:%M:%S")
# We store UTC timestamp
time_difference = datetime.datetime.utcnow() - insert_time
# Assert
self.assertEquals(time_difference.days, 0)
self.assertLess(
time_difference.seconds, 10,
"Difference in stored and retrieved time is too large.")
@blocking_call_on_reactor_thread
def set_db_version(self, version):
self.db.executescript(
u"UPDATE option SET value = '%d' WHERE key = 'database_version';" %
version)
self.db.close(commit=True)
self.db = MultiChainDB(self.getStateDir())
@blocking_call_on_reactor_thread
def test_database_upgrade(self):
self.set_db_version(1)
version, = next(
self.db.execute(
u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1"
))
self.assertEqual(version, u"3")
@blocking_call_on_reactor_thread
def test_database_create(self):
self.set_db_version(0)
version, = next(
self.db.execute(
u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1"
))
self.assertEqual(version, u"3")
@blocking_call_on_reactor_thread
def test_database_no_downgrade(self):
self.set_db_version(200000)
version, = next(
self.db.execute(
u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1"
))
self.assertEqual(version, u"200000")
@blocking_call_on_reactor_thread
def test_block_to_dictionary(self):
"""
Test whether a block is correctly represented when converted to a dictionary
"""
block_dict = dict(self.block1)
self.assertEqual(block_dict["up"], self.block1.up)
self.assertEqual(block_dict["down"], self.block1.down)
self.assertEqual(block_dict["insert_time"], self.block1.insert_time)
class TestDatabase(MultiChainTestCase):
"""
Tests the Database for MultiChain community.
Also tests integration with Dispersy.
This integration slows down the tests,
but can probably be removed and a Mock Dispersy could be used.
"""
def __init__(self, *args, **kwargs):
super(TestDatabase, self).__init__(*args, **kwargs)
@blocking_call_on_reactor_thread
@inlineCallbacks
def setUp(self, **kwargs):
yield super(TestDatabase, self).setUp()
path = os.path.join(self.getStateDir(), DATABASE_DIRECTORY)
if not os.path.exists(path):
os.makedirs(path)
self.db = MultiChainDB(None, self.getStateDir())
self.block1 = TestBlock()
self.block2 = TestBlock()
@blocking_call_on_reactor_thread
def test_add_block(self):
# Act
self.db.add_block(self.block1)
# Assert
result = self.db.get_by_hash_requester(self.block1.hash_requester)
self.assertEqual_block(self.block1, result)
@blocking_call_on_reactor_thread
def test_get_blocks(self):
"""
Test whether the right blocks are returned when fetching blocks
"""
self.db.add_block(self.block1)
self.assertEqual(
len(self.db.get_blocks(self.block1.public_key_requester)), 1)
self.block1.hash_requester = "a" * 40
self.db.add_block(self.block1)
self.assertEqual(
len(self.db.get_blocks(self.block1.public_key_requester)), 2)
self.assertEqual(
len(self.db.get_blocks(self.block1.public_key_requester, limit=1)),
1)
@blocking_call_on_reactor_thread
def test_get_num_interactors(self):
"""
Test whether the right number of interactors is returned
"""
crypto = ECCrypto()
my_key = crypto.key_to_bin(crypto.generate_key(u"curve25519").pub())
block1 = TestBlock()
block1.public_key_requester = my_key
block1.up = 100
block1.down = 100
self.db.add_block(block1)
block2 = TestBlock()
block2.public_key_responder = my_key
block2.up = 100
block2.down = 100
self.db.add_block(block2)
self.assertEqual((2, 2), self.db.get_num_unique_interactors(my_key))
@blocking_call_on_reactor_thread
def test_get_by_hash(self):
# Act
self.db.add_block(self.block1)
# Assert
result1 = self.db.get_by_hash_requester(self.block1.hash_requester)
result2 = self.db.get_by_hash(self.block1.hash_requester)
result3 = self.db.get_by_hash(self.block1.hash_responder)
self.assertEqual_block(self.block1, result1)
self.assertEqual_block(self.block1, result2)
self.assertEqual_block(self.block1, result3)
@blocking_call_on_reactor_thread
def test_add_two_blocks(self):
# Act
self.db.add_block(self.block1)
self.db.add_block(self.block2)
# Assert
result = self.db.get_by_hash_requester(self.block2.hash_requester)
self.assertEqual_block(self.block2, result)
@blocking_call_on_reactor_thread
def test_get_block_non_existing(self):
# Act
result = self.db.get_by_hash_requester(self.block1.hash_requester)
# Assert
self.assertEqual(None, result)
@blocking_call_on_reactor_thread
def test_contains_block_id_positive(self):
# Act
self.db.add_block(self.block1)
# Assert
self.assertTrue(self.db.contains(self.block1.hash_requester))
@blocking_call_on_reactor_thread
def test_contains_block_id_negative(self):
# Act & Assert
self.assertFalse(self.db.contains("NON EXISTING ID"))
@blocking_call_on_reactor_thread
def test_get_latest_sequence_number_not_existing(self):
# Act & Assert
self.assertEquals(
self.db.get_latest_sequence_number("NON EXISTING KEY"), -1)
@blocking_call_on_reactor_thread
def test_get_latest_sequence_number_public_key_requester(self):
# Arrange
# Make sure that there is a responder block with a lower sequence number.
# To test that it will look for both responder and requester.
self.db.add_block(self.block1)
self.block2.public_key_responder = self.block1.public_key_requester
self.block2.sequence_number_responder = self.block1.sequence_number_requester - 5
self.db.add_block(self.block2)
# Act & Assert
self.assertEquals(
self.db.get_latest_sequence_number(
self.block1.public_key_requester),
self.block1.sequence_number_requester)
@blocking_call_on_reactor_thread
def test_get_latest_sequence_number_public_key_responder(self):
# Arrange
# Make sure that there is a requester block with a lower sequence number.
# To test that it will look for both responder and requester.
self.db.add_block(self.block1)
self.block2.public_key_requester = self.block1.public_key_responder
self.block2.sequence_number_requester = self.block1.sequence_number_responder - 5
self.db.add_block(self.block2)
# Act & Assert
self.assertEquals(
self.db.get_latest_sequence_number(
self.block1.public_key_responder),
self.block1.sequence_number_responder)
@blocking_call_on_reactor_thread
def test_get_previous_id_not_existing(self):
# Act & Assert
self.assertEquals(self.db.get_latest_hash("NON EXISTING KEY"), None)
@blocking_call_on_reactor_thread
def test_get_previous_hash_of_requester(self):
# Arrange
# Make sure that there is a responder block with a lower sequence number.
# To test that it will look for both responder and requester.
self.db.add_block(self.block1)
self.block2.public_key_responder = self.block1.public_key_requester
self.block2.sequence_number_responder = self.block1.sequence_number_requester + 1
self.db.add_block(self.block2)
# Act & Assert
self.assertEquals(
self.db.get_latest_hash(self.block2.public_key_responder),
self.block2.hash_responder)
@blocking_call_on_reactor_thread
def test_get_previous_hash_of_responder(self):
# Arrange
# Make sure that there is a requester block with a lower sequence number.
# To test that it will look for both responder and requester.
self.db.add_block(self.block1)
self.block2.public_key_requester = self.block1.public_key_responder
self.block2.sequence_number_requester = self.block1.sequence_number_responder + 1
self.db.add_block(self.block2)
# Act & Assert
self.assertEquals(
self.db.get_latest_hash(self.block2.public_key_requester),
self.block2.hash_requester)
@blocking_call_on_reactor_thread
def test_get_by_sequence_number_by_mid_not_existing(self):
# Act & Assert
self.assertEquals(
self.db.get_by_public_key_and_sequence_number(
"NON EXISTING KEY", 0), None)
@blocking_call_on_reactor_thread
def test_get_by_public_key_and_sequence_number_requester(self):
# Arrange
# Make sure that there is a responder block with a lower sequence number.
# To test that it will look for both responder and requester.
self.db.add_block(self.block1)
# Act & Assert
self.assertEqual_block(
self.block1,
self.db.get_by_public_key_and_sequence_number(
self.block1.public_key_requester,
self.block1.sequence_number_requester))
@blocking_call_on_reactor_thread
def test_get_by_public_key_and_sequence_number_responder(self):
# Arrange
# Make sure that there is a responder block with a lower sequence number.
# To test that it will look for both responder and requester.
self.db.add_block(self.block1)
# Act & Assert
self.assertEqual_block(
self.block1,
self.db.get_by_public_key_and_sequence_number(
self.block1.public_key_responder,
self.block1.sequence_number_responder))
@blocking_call_on_reactor_thread
def test_get_total(self):
# Arrange
self.db.add_block(self.block1)
self.block2.public_key_requester = self.block1.public_key_responder
self.block2.sequence_number_requester = self.block1.sequence_number_responder + 1
self.block2.total_up_requester = self.block1.total_up_responder + self.block2.up
self.block2.total_down_requester = self.block1.total_down_responder + self.block2.down
self.db.add_block(self.block2)
# Act
(result_up,
result_down) = self.db.get_total(self.block2.public_key_requester)
# Assert
self.assertEqual(self.block2.total_up_requester, result_up)
self.assertEqual(self.block2.total_down_requester, result_down)
@blocking_call_on_reactor_thread
def test_get_total_not_existing(self):
# Arrange
self.db.add_block(self.block1)
# Act
(result_up,
result_down) = self.db.get_total(self.block2.public_key_requester)
# Assert
self.assertEqual(0, result_up)
self.assertEqual(0, result_down)
@blocking_call_on_reactor_thread
def test_save_large_upload_download_block(self):
"""
Test if the block can save very large numbers.
"""
# Arrange
self.block1.total_up_requester = long(pow(2, 62))
self.block1.total_down_requester = long(pow(2, 62))
self.block1.total_up_responder = long(pow(2, 61))
self.block1.total_down_responder = pow(2, 60)
# Act
self.db.add_block(self.block1)
# Assert
result = self.db.get_by_hash(self.block1.hash_requester)
self.assertEqual_block(self.block1, result)
@blocking_call_on_reactor_thread
def test_get_insert_time(self):
# Arrange
# Upon adding the block to the database, the timestamp will get added.
self.db.add_block(self.block1)
# Act
# Retrieving the block from the database will result in a block with a
# timestamp
result = self.db.get_by_hash(self.block1.hash_requester)
insert_time = datetime.datetime.strptime(result.insert_time,
"%Y-%m-%d %H:%M:%S")
# We store UTC timestamp
time_difference = datetime.datetime.utcnow() - insert_time
# Assert
self.assertEquals(time_difference.days, 0)
self.assertLess(
time_difference.seconds, 10,
"Difference in stored and retrieved time is too large.")
@blocking_call_on_reactor_thread
def set_db_version(self, version):
self.db.executescript(
u"UPDATE option SET value = '%d' WHERE key = 'database_version';" %
version)
self.db.close(commit=True)
self.db = MultiChainDB(None, self.getStateDir())
@blocking_call_on_reactor_thread
def test_database_upgrade(self):
self.set_db_version(1)
version, = next(
self.db.execute(
u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1"
))
self.assertEqual(version, u"2")
@blocking_call_on_reactor_thread
def test_database_create(self):
self.set_db_version(0)
version, = next(
self.db.execute(
u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1"
))
self.assertEqual(version, u"2")
@blocking_call_on_reactor_thread
def test_database_no_downgrade(self):
self.set_db_version(200000)
version, = next(
self.db.execute(
u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1"
))
self.assertEqual(version, u"200000")
@blocking_call_on_reactor_thread
def test_block_to_dictionary(self):
"""
Test whether a block is correctly represented when converted to a dictionary
"""
block_dict = self.block1.to_dictionary()
self.assertEqual(block_dict["up"], self.block1.up)
self.assertEqual(block_dict["down"], self.block1.down)
self.assertEqual(block_dict["insert_time"], self.block1.insert_time)
class MultiChainCommunity(Community):
"""
Community for reputation based on MultiChain tamper proof interaction history.
"""
def __init__(self, *args, **kwargs):
super(MultiChainCommunity, self).__init__(*args, **kwargs)
self.logger = logging.getLogger(self.__class__.__name__)
self.notifier = None
self.persistence = MultiChainDB(self.dispersy.working_directory)
# We store the bytes send and received in the tunnel community in a dictionary.
# The key is the public key of the peer being interacted with, the value a tuple of the up and down bytes
# This data is not used to create outgoing requests, but _only_ to verify incoming requests
self.pending_bytes = dict()
self.logger.debug(
"The multichain community started with Public Key: %s",
self.my_member.public_key.encode("hex"))
def initialize(self, tribler_session=None):
super(MultiChainCommunity, self).initialize()
if tribler_session:
self.notifier = tribler_session.notifier
self.notifier.add_observer(self.on_tunnel_remove, NTFY_TUNNEL,
[NTFY_REMOVE])
@classmethod
def get_master_members(cls, dispersy):
# generated: Sun Apr 23 10:06:29 2017
# curve: None
# len: 571 bits ~ 144 bytes signature
# pub: 170 3081a7301006072a8648ce3d020106052b8104002703819200040503dac58c19267f12cb0cf667e480816cd2574acae5293b5
# 9d7c3da32e02b4747f7e2e9e9c880d2e5e2ba8b7fcc9892cb39b797ef98483ffd58739ed20990f8e3df7d1ec5a7ad2c0338dc206c4383a
# 943e3e2c682ac4b585880929a947ffd50057b575fc30ec88eada3ce6484e5e4d6fdf41984cd1e51aaacc5f9a51bcc8393aea1f786fc47c
# bf994cb1339f706df4a
# pub-sha1 b78a5e252bf2f7be8716c383734f325b9aaff844
# -----BEGIN PUBLIC KEY-----
# MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQFA9rFjBkmfxLLDPZn5ICBbNJXSsrl
# KTtZ18PaMuArR0f34unpyIDS5eK6i3/MmJLLObeX75hIP/1Yc57SCZD44999HsWn
# rSwDONwgbEODqUPj4saCrEtYWICSmpR//VAFe1dfww7Ijq2jzmSE5eTW/fQZhM0e
# UaqsxfmlG8yDk66h94b8R8v5lMsTOfcG30o=
# -----END PUBLIC KEY-----
master_key = "3081a7301006072a8648ce3d020106052b8104002703819200040503dac58c19267f12cb0cf667e480816cd2574acae" \
"5293b59d7c3da32e02b4747f7e2e9e9c880d2e5e2ba8b7fcc9892cb39b797ef98483ffd58739ed20990f8e3df7d1ec5" \
"a7ad2c0338dc206c4383a943e3e2c682ac4b585880929a947ffd50057b575fc30ec88eada3ce6484e5e4d6fdf41984c" \
"d1e51aaacc5f9a51bcc8393aea1f786fc47cbf994cb1339f706df4a"
return [dispersy.get_member(public_key=master_key.decode("HEX"))]
def initiate_meta_messages(self):
"""
Setup all message that can be received by this community and the super classes.
:return: list of meta messages.
"""
return super(MultiChainCommunity, self).initiate_meta_messages() + [
Message(self, HALF_BLOCK, NoAuthentication(), PublicResolution(),
DirectDistribution(), CandidateDestination(),
HalfBlockPayload(), self._generic_timeline_check,
self.received_half_block),
Message(self, CRAWL, MemberAuthentication(), PublicResolution(),
DirectDistribution(), CandidateDestination(),
CrawlRequestPayload(), self._generic_timeline_check,
self.received_crawl_request)
]
def initiate_conversions(self):
return [DefaultConversion(self), MultiChainConversion(self)]
def send_block(self, candidate, block):
if candidate.get_member():
self.logger.debug(
"Sending block to %s (%s)",
candidate.get_member().public_key.encode("hex")[-8:], block)
else:
self.logger.debug("Sending block to %s (%s)", candidate, block)
message = self.get_meta_message(HALF_BLOCK).impl(
authentication=tuple(),
distribution=(self.claim_global_time(), ),
destination=(candidate, ),
payload=(block, ))
try:
self.dispersy.store_update_forward([message], False, False, True)
except DelayPacketByMissingMember:
self.logger.warn(
"Missing member in MultiChain community to send signature request to"
)
def sign_block(self,
candidate,
public_key=None,
bytes_up=None,
bytes_down=None,
linked=None):
"""
Create, sign, persist and send a block signed message
:param candidate: The peer with whom you have interacted, as a dispersy candidate
:param bytes_up: The bytes you have uploaded to the peer in this interaction
:param bytes_down: The bytes you have downloaded from the peer in this interaction
:param linked: The block that the requester is asking us to sign
"""
# NOTE to the future: This method reads from the database, increments and then writes back. If in some future
# this method is allowed to execute in parallel, be sure to lock from before .create up to after .add_block
assert bytes_up is None and bytes_down is None and linked is not None or \
bytes_up is not None and bytes_down is not None and linked is None, \
"Either provide a linked block or byte counts, not both"
assert linked is None or linked.link_public_key == self.my_member.public_key, \
"Cannot counter sign block not addressed to self"
assert linked is None or linked.link_sequence_number == UNKNOWN_SEQ, \
"Cannot counter sign block that is not a request"
block = MultiChainBlock.create(self.persistence,
self.my_member.public_key, linked)
if linked is None:
block.up = bytes_up
block.down = bytes_down
block.total_up += bytes_up
block.total_down += bytes_down
block.link_public_key = public_key
block.sign(self.my_member.private_key)
validation = block.validate(self.persistence)
self.logger.info("Signed block to %s (%s) validation result %s",
block.link_public_key.encode("hex")[-8:], block,
validation)
if validation[0] != ValidationResult.partial_next and validation[
0] != ValidationResult.valid:
self.logger.error("Signed block did not validate?! Result %s",
repr(validation))
else:
self.persistence.add_block(block)
self.send_block(candidate, block)
def received_half_block(self, messages):
"""
We've received a half block, either because we sent a SIGNED message to some one or we are crawling
:param messages The half block messages
"""
self.logger.debug("Received %d half block messages.", len(messages))
for message in messages:
blk = message.payload.block
validation = blk.validate(self.persistence)
self.logger.debug("Block validation result %s, %s, (%s)",
validation[0], validation[1], blk)
if validation[0] == ValidationResult.invalid:
continue
elif not self.persistence.contains(blk):
self.persistence.add_block(blk)
else:
self.logger.debug("Received already known block (%s)", blk)
# Is this a request, addressed to us, and have we not signed it already?
if blk.link_sequence_number != UNKNOWN_SEQ or \
blk.link_public_key != self.my_member.public_key or \
self.persistence.get_linked(blk) is not None:
continue
self.logger.info("Received request block addressed to us (%s)",
blk)
# determine if we want to sign (i.e. the requesting public key has enough pending bytes)
pend = self.pending_bytes.get(blk.public_key)
if not pend or not pend.add(-blk.down, -blk.up):
self.logger.info(
"Request block counter party does not have enough bytes pending."
)
continue
crawl_task = "crawl_%s" % blk.hash
# It is important that the request matches up with its previous block, gaps cannot be tolerated at
# this point. We already dropped invalids, so here we delay this message if the result is partial,
# partial_previous or no-info. We send a crawl request to the requester to (hopefully) close the gap
if validation[0] == ValidationResult.partial_previous or validation[0] == ValidationResult.partial or \
validation[0] == ValidationResult.no_info:
self.logger.info(
"Request block could not be validated sufficiently, crawling requester. %s",
validation)
# Note that this code does not cover the scenario where we obtain this block indirectly.
# We modified the counters to get here, correct pending bytes since we did not really sign the block yet
pend.add(blk.down, blk.up)
# Are we already waiting for this crawl to happen?
# For example: it's taking longer than 5 secs or the block message reached us twice via different paths
if self.is_pending_task_active(crawl_task):
continue
self.send_crawl_request(
message.candidate, blk.public_key,
max(GENESIS_SEQ, blk.sequence_number - 5))
# Make sure we get called again after a while. Note that the cleanup task on pend will prevent
# us from waiting on the peer forever.
self.register_task(
crawl_task,
reactor.callLater(5.0, self.received_half_block,
[message]))
else:
self.sign_block(message.candidate, linked=blk)
if self.is_pending_task_active(crawl_task):
self.cancel_pending_task(crawl_task)
continue
def send_crawl_request(self, candidate, public_key, sequence_number=None):
sq = sequence_number
if sequence_number is None:
blk = self.persistence.get_latest(public_key)
sq = blk.sequence_number if blk else GENESIS_SEQ
sq = max(GENESIS_SEQ, sq)
self.logger.info("Requesting crawl of node %s:%d",
public_key.encode("hex")[-8:], sq)
message = self.get_meta_message(CRAWL).impl(
authentication=(self.my_member, ),
distribution=(self.claim_global_time(), ),
destination=(candidate, ),
payload=(sq, ))
self.dispersy.store_update_forward([message], False, False, True)
def received_crawl_request(self, messages):
self.logger.debug("Received %d crawl messages.", len(messages))
for message in messages:
self.logger.info(
"Received crawl request from node %s for sequence number %d",
message.candidate.get_member().public_key.encode("hex")[-8:],
message.payload.requested_sequence_number)
blocks = self.persistence.crawl(
self.my_member.public_key,
message.payload.requested_sequence_number)
count = len(blocks)
for blk in blocks:
self.send_block(message.candidate, blk)
self.logger.info("Sent %d blocks", count)
@blocking_call_on_reactor_thread
def get_statistics(self, public_key=None):
"""
Returns a dictionary with some statistics regarding the local multichain database
:returns a dictionary with statistics
"""
if public_key is None:
public_key = self.my_member.public_key
latest_block = self.persistence.get_latest(public_key)
statistics = dict()
statistics["id"] = public_key.encode("hex")
interacts = self.persistence.get_num_unique_interactors(public_key)
statistics["peers_that_pk_helped"] = interacts[0] if interacts[
0] is not None else 0
statistics["peers_that_helped_pk"] = interacts[1] if interacts[
1] is not None else 0
if latest_block:
statistics["total_blocks"] = latest_block.sequence_number
statistics["total_up"] = latest_block.total_up
statistics["total_down"] = latest_block.total_down
statistics["latest_block"] = dict(latest_block)
else:
statistics["total_blocks"] = 0
statistics["total_up"] = 0
statistics["total_down"] = 0
return statistics
@inlineCallbacks
def unload_community(self):
self.logger.debug("Unloading the MultiChain Community.")
if self.notifier:
self.notifier.remove_observer(self.on_tunnel_remove)
for pk in self.pending_bytes:
if self.pending_bytes[pk].clean is not None:
self.pending_bytes[pk].clean.reset(0)
yield super(MultiChainCommunity, self).unload_community()
# Close the persistence layer
self.persistence.close()
@forceDBThread
def on_tunnel_remove(self, subject, change_type, tunnel, candidate):
"""
Handler for the remove event of a tunnel. This function will attempt to create a block for the amounts that
were transferred using the tunnel.
:param subject: Category of the notifier event
:param change_type: Type of the notifier event
:param tunnel: The tunnel that was removed (closed)
:param candidate: The dispersy candidate with whom this node has interacted in the tunnel
"""
from Tribler.community.tunnel.tunnel_community import Circuit, RelayRoute, TunnelExitSocket
assert isinstance(tunnel, Circuit) or isinstance(tunnel, RelayRoute) or isinstance(tunnel, TunnelExitSocket), \
"on_tunnel_remove() was called with an object that is not a Circuit, RelayRoute or TunnelExitSocket"
assert isinstance(tunnel.bytes_up, int) and isinstance(tunnel.bytes_down, int),\
"tunnel instance must provide byte counts in int"
up = tunnel.bytes_up
down = tunnel.bytes_down
pk = candidate.get_member().public_key
# If the transaction is not big enough we discard the bytes up and down.
if up + down >= MIN_TRANSACTION_SIZE:
# Tie breaker to prevent both parties from requesting
if up > down or (up == down and self.my_member.public_key > pk):
self.register_task(
"sign_%s" % tunnel.circuit_id,
reactor.callLater(5, self.sign_block, candidate, pk,
tunnel.bytes_up, tunnel.bytes_down))
else:
pend = self.pending_bytes.get(pk)
if not pend:
self.pending_bytes[pk] = PendingBytes(
up, down,
reactor.callLater(2 * 60, self.cleanup_pending, pk))
else:
pend.add(up, down)
def cleanup_pending(self, public_key):
self.pending_bytes.pop(public_key, None)
class MultiChainCommunity(Community):
"""
Community for reputation based on MultiChain tamper proof interaction history.
"""
def __init__(self, *args, **kwargs):
super(MultiChainCommunity, self).__init__(*args, **kwargs)
self.logger = logging.getLogger(self.__class__.__name__)
self.notifier = None
self._private_key = self.my_member.private_key
self._public_key = self.my_member.public_key
self.persistence = MultiChainDB(self.dispersy,
self.dispersy.working_directory)
self.logger.debug(
"The multichain community started with Public Key: %s",
base64.encodestring(self._public_key))
# No response is expected yet.
self.expected_response = None
def initialize(self, tribler_session=None):
super(MultiChainCommunity, self).initialize()
if tribler_session:
self.notifier = tribler_session.notifier
self.notifier.add_observer(self.on_tunnel_remove, NTFY_TUNNEL,
[NTFY_REMOVE])
@classmethod
def get_master_members(cls, dispersy):
# generated: Fri Jul 1 15:22:20 2016
# curve: None
# len:571 bits ~ 144 bytes signature
# pub: 170 3081a7301006072a8648ce3d020106052b81040027038192000407afa96c83660dccfbf02a45b68f4bc
# 4957539860a3fe1ad4a18ccbfc2a60af1174e1f5395a7917285d09ab67c3d80c56caf5396fc5b231d84ceac23627
# 930b4c35cbfce63a49805030dabbe9b5302a966b80eefd7003a0567c65ccec5ecde46520cfe1875b1187d469823d
# 221417684093f63c33a8ff656331898e4bc853bcfaac49bc0b2a99028195b7c7dca0aea65
# pub-sha1 15ade4f5fb0f0f8019d8430473aaba4305e61753
# -----BEGIN PUBLIC KEY-----
# MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQHr6lsg2YNzPvwKkW2j0vElXU5hgo/4a1KGMy/wqYK8RdOH1OVp5FyhdCatnw9g
# MVsr1OW/FsjHYTOrCNieTC0w1y/zmOkmAUDDau+m1MCqWa4Du/XADoFZ8ZczsXs3kZSDP4YdbEYfUaYI9IhQXaECT9jwzqP9l
# YzGJjkvIU7z6rEm8CyqZAoGVt8fcoK6mU=
# -----END PUBLIC KEY-----
master_key = "3081a7301006072a8648ce3d020106052b81040027038192000407afa96c83660dccfbf02a45b68f4bc" + \
"4957539860a3fe1ad4a18ccbfc2a60af1174e1f5395a7917285d09ab67c3d80c56caf5396fc5b231d84ceac23627" + \
"930b4c35cbfce63a49805030dabbe9b5302a966b80eefd7003a0567c65ccec5ecde46520cfe1875b1187d469823d" + \
"221417684093f63c33a8ff656331898e4bc853bcfaac49bc0b2a99028195b7c7dca0aea65"
master_key_hex = master_key.decode("HEX")
master = dispersy.get_member(public_key=master_key_hex)
return [master]
def initiate_meta_messages(self):
"""
Setup all message that can be received by this community and the super classes.
:return: list of meta messages.
"""
return super(MultiChainCommunity, self).initiate_meta_messages() + [
Message(
self, SIGNATURE,
DoubleMemberAuthentication(
allow_signature_func=self.allow_signature_request,
split_payload_func=split_function), PublicResolution(),
DirectDistribution(), CandidateDestination(),
SignaturePayload(), self._generic_timeline_check,
self.received_signature_response),
Message(self, CRAWL_REQUEST, MemberAuthentication(),
PublicResolution(), DirectDistribution(),
CandidateDestination(), CrawlRequestPayload(),
self._generic_timeline_check, self.received_crawl_request),
Message(self, CRAWL_RESPONSE, MemberAuthentication(),
PublicResolution(), DirectDistribution(),
CandidateDestination(), CrawlResponsePayload(),
self._generic_timeline_check,
self.received_crawl_response),
Message(self, CRAWL_RESUME, MemberAuthentication(),
PublicResolution(), DirectDistribution(),
CandidateDestination(), CrawlResumePayload(),
self._generic_timeline_check,
self.received_crawl_resumption)
]
def initiate_conversions(self):
return [DefaultConversion(self), MultiChainConversion(self)]
def schedule_block(self, candidate, bytes_up, bytes_down):
"""
Schedule a block for the current outstanding amounts
:param candidate: The peer with whom you have interacted, as a dispersy candidate
:param bytes_up: The bytes you have uploaded to the peer in this interaction
:param bytes_down: The bytes you have downloaded from the peer in this interaction
"""
self.logger.info("MULTICHAIN: Schedule Block called. Candidate: " +
str(candidate) + " UP: " + str(bytes_up) + " DOWN: " +
str(bytes_down))
self.add_discovered_candidate(candidate)
if candidate and candidate.get_member():
# Convert to MB
total_amount_sent_mb = bytes_up / MEGA_DIVIDER
total_amount_received_mb = bytes_down / MEGA_DIVIDER
# Try to send the request
try:
self.publish_signature_request_message(
candidate, total_amount_sent_mb, total_amount_received_mb)
except DelayPacketByMissingMember:
self.logger.warn(
"Missing member in MultiChain community to send signature request to"
)
else:
self.logger.warn(
"No valid candidate found for: %s to request block from.",
candidate)
def publish_signature_request_message(self, candidate, up, down):
"""
Creates and sends out a signed signature_request message
Returns true upon success
:param candidate: The candidate that the signature request will be sent to.
:param (int) up: The amount of Megabytes that have been sent to the candidate that need to signed.
:param (int) down: The amount of Megabytes that have been received from the candidate that need to signed.
return (bool) if request is sent.
"""
message = self.create_signature_request_message(candidate, up, down)
self.create_signature_request(candidate, message,
self.allow_signature_response)
self.persist_signature_request(message)
return True
def create_signature_request_message(self, candidate, up, down):
"""
Create a signature request message using the current time stamp.
:param candidate: The candidate that the signature request will be sent to.
:param (int) up: The amount of Megabytes that have been sent to the candidate that need to signed.
:param (int) down: The amount of Megabytes that have been received from the candidate that need to signed.
:return: Signature_request message ready for distribution.
"""
# Instantiate the data
total_up_requester, total_down_requester = self._get_next_total(
up, down)
# Instantiate the personal information
sequence_number_requester = self._get_next_sequence_number()
previous_hash_requester = self._get_latest_hash()
payload = (up, down, total_up_requester, total_down_requester,
sequence_number_requester, previous_hash_requester)
meta = self.get_meta_message(SIGNATURE)
message = meta.impl(
authentication=([self.my_member,
candidate.get_member()], ),
distribution=(self.claim_global_time(), ),
payload=payload)
return message
def allow_signature_request(self, message):
"""
We've received a signature request message, we must either:
a. Create and sign the response part of the message, send it back, and persist the block.
b. Drop the message. (Future work: notify the sender of dropping)
:param message The message containing the received signature request.
"""
self.logger.info("Received signature request for: [Up = " +
str(message.payload.up) + "MB | Down = " +
str(message.payload.down) + " MB]")
# TODO: This code always signs a request. Checks and rejects should be inserted here!
# TODO: Like basic total_up == previous_total_up + block.up or more sophisticated chain checks.
payload = message.payload
# The up and down values are reversed for the responder.
total_up_responder, total_down_responder = self._get_next_total(
payload.down, payload.up)
sequence_number_responder = self._get_next_sequence_number()
previous_hash_responder = self._get_latest_hash()
payload = (payload.up, payload.down, payload.total_up_requester,
payload.total_down_requester,
payload.sequence_number_requester,
payload.previous_hash_requester, total_up_responder,
total_down_responder, sequence_number_responder,
previous_hash_responder)
meta = self.get_meta_message(SIGNATURE)
message = meta.impl(authentication=(message.authentication.members,
message.authentication.signatures),
distribution=(message.distribution.global_time, ),
payload=payload)
self.persist_signature_response(message)
self.logger.info("Sending signature response.")
return message
def allow_signature_response(self, request, response, modified):
"""
We've received a signature response message after sending a request, we must return either:
a. True, if we accept this message
b. False, if not (because of inconsistencies in the payload)
:param request The original message as send by this node
:param response The response message received
:param modified (bool) True if the message was modified
"""
if not response:
self.logger.info("Timeout received for signature request.")
return False
else:
# TODO: Check whether we are expecting a response
self.logger.info("Signature response received. Modified: %s",
modified)
return (request.payload.sequence_number_requester
== response.payload.sequence_number_requester
and request.payload.previous_hash_requester
== response.payload.previous_hash_requester and modified)
def received_signature_response(self, messages):
"""
We've received a valid signature response and must process this message.
:param messages The received, and validated signature response messages
"""
self.logger.info("Valid %s signature response(s) received.",
len(messages))
for message in messages:
self.update_signature_response(message)
def persist_signature_response(self, message):
"""
Persist the signature response message, when this node has not yet persisted the corresponding request block.
A hash will be created from the message and this will be used as an unique identifier.
:param message:
"""
block = DatabaseBlock.from_signature_response_message(message)
self.logger.info("Persisting sr: %s",
base64.encodestring(block.hash_requester).strip())
self.persistence.add_block(block)
def update_signature_response(self, message):
"""
Update the signature response message, when this node has already persisted the corresponding request block.
A hash will be created from the message and this will be used as an unique identifier.
:param message:
"""
block = DatabaseBlock.from_signature_response_message(message)
self.logger.info("Persisting sr: %s",
base64.encodestring(block.hash_requester).strip())
self.persistence.update_block_with_responder(block)
def persist_signature_request(self, message):
"""
Persist the signature request message as a block.
The block will be updated when a response is received.
:param message:
"""
block = DatabaseBlock.from_signature_request_message(message)
self.logger.info("Persisting sr: %s",
base64.encodestring(block.hash_requester).strip())
self.persistence.add_block(block)
def send_crawl_request(self, candidate, sequence_number=None):
if sequence_number is None:
sequence_number = self.persistence.get_latest_sequence_number(
candidate.get_member().public_key)
self.logger.info(
"Crawler: Requesting crawl from node %s, from sequence number %d",
base64.encodestring(candidate.get_member().mid).strip(),
sequence_number)
meta = self.get_meta_message(CRAWL_REQUEST)
message = meta.impl(authentication=(self.my_member, ),
distribution=(self.claim_global_time(), ),
destination=(candidate, ),
payload=(sequence_number, ))
self.dispersy.store_update_forward([message], False, False, True)
def received_crawl_request(self, messages):
for message in messages:
self.logger.info(
"Crawler: Received crawl request from node %s, from sequence number %d",
base64.encodestring(
message.candidate.get_member().mid).strip(),
message.payload.requested_sequence_number)
self.crawl_requested(message.candidate,
message.payload.requested_sequence_number)
def crawl_requested(self, candidate, sequence_number):
blocks = self.persistence.get_blocks_since(self._public_key,
sequence_number)
if len(blocks) > 0:
self.logger.debug("Crawler: Sending %d blocks", len(blocks))
messages = [
self.get_meta_message(CRAWL_RESPONSE).impl(
authentication=(self.my_member, ),
distribution=(self.claim_global_time(), ),
destination=(candidate, ),
payload=block.to_payload()) for block in blocks
]
self.dispersy.store_update_forward(messages, False, False, True)
if len(blocks) > 1:
# we sent more than 1 block. Send a resumption token so the other side knows it should continue crawling
# last_block = blocks[-1]
# resumption_number = last_block.sequence_number_requster if
# last_block.mid_requester == self._mid else last_block.sequence_number_responder
message = self.get_meta_message(CRAWL_RESUME).impl(
authentication=(self.my_member, ),
distribution=(self.claim_global_time(), ),
destination=(candidate, ),
# payload=(resumption_number))
payload=())
self.dispersy.store_update_forward([message], False, False,
True)
else:
# This is slightly worrying since the last block should always be returned.
# Or rather, the other side is requesting blocks starting from a point in the future.
self.logger.info("Crawler: No blocks")
def received_crawl_response(self, messages):
self.logger.debug("Crawler: Valid %d block response(s) received.",
len(messages))
for message in messages:
requester = self.dispersy.get_member(
public_key=message.payload.public_key_requester)
responder = self.dispersy.get_member(
public_key=message.payload.public_key_responder)
block = DatabaseBlock.from_block_response_message(
message, requester, responder)
# Create the hash of the message
if not self.persistence.contains(block.hash_requester):
self.logger.info(
"Crawler: Persisting sr: %s from ip (%s:%d)",
base64.encodestring(block.hash_requester).strip(),
message.candidate.sock_addr[0],
message.candidate.sock_addr[1])
self.persistence.add_block(block)
else:
self.logger.debug("Crawler: Received already known block")
def received_crawl_resumption(self, messages):
self.logger.info("Crawler: Valid %s crawl resumptions received.",
len(messages))
for message in messages:
self.send_crawl_request(message.candidate)
@blocking_call_on_reactor_thread
def get_statistics(self):
"""
Returns a dictionary with some statistics regarding the local multichain database
:returns a dictionary with statistics
"""
statistics = dict()
statistics["self_id"] = base64.encodestring(self._public_key).strip()
statistics[
"self_total_blocks"] = self.persistence.get_latest_sequence_number(
self._public_key)
(statistics["self_total_up_mb"],
statistics["self_total_down_mb"]) = self.persistence.get_total(
self._public_key)
(statistics["self_peers_helped"], statistics["self_peers_helped_you"]
) = self.persistence.get_num_unique_interactors(self._public_key)
latest_block = self.persistence.get_latest_block(self._public_key)
if latest_block:
statistics["latest_block_insert_time"] = str(
latest_block.insert_time)
statistics["latest_block_id"] = base64.encodestring(
latest_block.hash_requester).strip()
statistics["latest_block_requester_id"] = base64.encodestring(
latest_block.public_key_requester).strip()
statistics["latest_block_responder_id"] = base64.encodestring(
latest_block.public_key_responder).strip()
statistics["latest_block_up_mb"] = str(latest_block.up)
statistics["latest_block_down_mb"] = str(latest_block.down)
else:
statistics["latest_block_insert_time"] = ""
statistics["latest_block_id"] = ""
statistics["latest_block_requester_id"] = ""
statistics["latest_block_responder_id"] = ""
statistics["latest_block_up_mb"] = ""
statistics["latest_block_down_mb"] = ""
return statistics
def _get_next_total(self, up, down):
"""
Returns the next total numbers of up and down incremented with the current interaction up and down metric.
:param up: Up metric for the interaction.
:param down: Down metric for the interaction.
:return: (total_up (int), total_down (int)
"""
total_up, total_down = self.persistence.get_total(self._public_key)
return total_up + up, total_down + down
def _get_next_sequence_number(self):
return self.persistence.get_latest_sequence_number(
self._public_key) + 1
def _get_latest_hash(self):
previous_hash = self.persistence.get_latest_hash(self._public_key)
return previous_hash if previous_hash else GENESIS_ID
@inlineCallbacks
def unload_community(self):
self.logger.debug("Unloading the MultiChain Community.")
if self.notifier:
self.notifier.remove_observer(self.on_tunnel_remove)
yield super(MultiChainCommunity, self).unload_community()
# Close the persistence layer
self.persistence.close()
@forceDBThread
def on_tunnel_remove(self, subject, change_type, tunnel, candidate):
"""
Handler for the remove event of a tunnel. This function will attempt to create a block for the amounts that
were transferred using the tunnel.
:param subject: Category of the notifier event
:param change_type: Type of the notifier event
:param tunnel: The tunnel that was removed (closed)
:param candidate: The dispersy candidate with whom this node has interacted in the tunnel
"""
from Tribler.community.tunnel.tunnel_community import Circuit, RelayRoute, TunnelExitSocket
assert isinstance(tunnel, Circuit) or isinstance(tunnel, RelayRoute) or isinstance(tunnel, TunnelExitSocket), \
"on_tunnel_remove() was called with an object that is not a Circuit, RelayRoute or TunnelExitSocket"
if isinstance(tunnel.bytes_up, int) and isinstance(
tunnel.bytes_down, int):
if tunnel.bytes_up > MEGA_DIVIDER or tunnel.bytes_down > MEGA_DIVIDER:
# Tie breaker to prevent both parties from requesting
if self._public_key > candidate.get_member().public_key:
self.schedule_block(candidate, tunnel.bytes_up,
tunnel.bytes_down)