def setUp(self, **kwargs): yield super(TestDatabase, self).setUp() path = os.path.join(self.getStateDir(), DATABASE_DIRECTORY) if not os.path.exists(path): os.makedirs(path) self.db = MultiChainDB(self.getStateDir()) self.block1 = TestBlock() self.block2 = TestBlock()
def __init__(self, *args, **kwargs): super(MultiChainCommunity, self).__init__(*args, **kwargs) self.logger = logging.getLogger(self.__class__.__name__) self.notifier = None self.persistence = MultiChainDB(self.dispersy.working_directory) # We store the bytes send and received in the tunnel community in a dictionary. # The key is the public key of the peer being interacted with, the value a tuple of the up and down bytes # This data is not used to create outgoing requests, but _only_ to verify incoming requests self.pending_bytes = dict() self.logger.debug( "The multichain community started with Public Key: %s", self.my_member.public_key.encode("hex"))
def __init__(self, *args, **kwargs): super(MultiChainCommunity, self).__init__(*args, **kwargs) self.logger = logging.getLogger(self.__class__.__name__) self.notifier = None self._private_key = self.my_member.private_key self._public_key = self.my_member.public_key self.persistence = MultiChainDB(self.dispersy, self.dispersy.working_directory) self.logger.debug( "The multichain community started with Public Key: %s", base64.encodestring(self._public_key)) # No response is expected yet. self.expected_response = None
def set_db_version(self, version): self.db.executescript( u"UPDATE option SET value = '%d' WHERE key = 'database_version';" % version) self.db.close(commit=True) self.db = MultiChainDB(self.getStateDir())
class TestDatabase(MultiChainTestCase): """ Tests the Database for MultiChain community. """ def __init__(self, *args, **kwargs): super(TestDatabase, self).__init__(*args, **kwargs) @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self, **kwargs): yield super(TestDatabase, self).setUp() path = os.path.join(self.getStateDir(), DATABASE_DIRECTORY) if not os.path.exists(path): os.makedirs(path) self.db = MultiChainDB(self.getStateDir()) self.block1 = TestBlock() self.block2 = TestBlock() @blocking_call_on_reactor_thread def test_add_block(self): # Act self.db.add_block(self.block1) # Assert result = self.db.get_latest(self.block1.public_key) self.assertEqual_block(self.block1, result) @blocking_call_on_reactor_thread def test_get_num_interactors(self): """ Test whether the right number of interactors is returned """ self.block2 = TestBlock(previous=self.block1) self.db.add_block(self.block1) self.db.add_block(self.block2) self.assertEqual( (2, 2), self.db.get_num_unique_interactors(self.block1.public_key)) @blocking_call_on_reactor_thread def test_add_two_blocks(self): # Act self.db.add_block(self.block1) self.db.add_block(self.block2) # Assert result = self.db.get_latest(self.block2.public_key) self.assertEqual_block(self.block2, result) @blocking_call_on_reactor_thread def test_get_block_non_existing(self): # Act result = self.db.get_latest(self.block1.public_key) # Assert self.assertEqual(None, result) @blocking_call_on_reactor_thread def test_contains_block_id_positive(self): # Act self.db.add_block(self.block1) # Assert self.assertTrue(self.db.contains(self.block1)) @blocking_call_on_reactor_thread def test_contains_block_id_negative(self): # Act & Assert self.assertFalse(self.db.contains(self.block1)) @blocking_call_on_reactor_thread def test_get_linked_forward(self): # Arrange self.block2 = TestBlock.create(self.db, self.block2.public_key, link=self.block1) self.db.add_block(self.block1) self.db.add_block(self.block2) # Act result = self.db.get_linked(self.block1) # Assert self.assertEqual_block(self.block2, result) @blocking_call_on_reactor_thread def test_get_linked_backwards(self): # Arrange self.block2 = TestBlock.create(self.db, self.block2.public_key, link=self.block1) self.db.add_block(self.block1) self.db.add_block(self.block2) # Act result = self.db.get_linked(self.block2) # Assert self.assertEqual_block(self.block1, result) @blocking_call_on_reactor_thread def test_get_block_after(self): # Arrange self.block2.public_key = self.block1.public_key self.block2.sequence_number = self.block1.sequence_number + 1 block3 = TestBlock() block3.public_key = self.block2.public_key block3.sequence_number = self.block2.sequence_number + 10 self.db.add_block(self.block1) self.db.add_block(self.block2) self.db.add_block(block3) # Act result = self.db.get_block_after(self.block2) # Assert self.assertEqual_block(block3, result) @blocking_call_on_reactor_thread def test_get_block_before(self): # Arrange self.block2.public_key = self.block1.public_key self.block2.sequence_number = self.block1.sequence_number + 1 block3 = TestBlock() block3.public_key = self.block2.public_key block3.sequence_number = self.block2.sequence_number + 10 self.db.add_block(self.block1) self.db.add_block(self.block2) self.db.add_block(block3) # Act result = self.db.get_block_before(self.block2) # Assert self.assertEqual_block(self.block1, result) @blocking_call_on_reactor_thread def test_save_large_upload_download_block(self): """ Test if the block can save very large numbers. """ # Arrange self.block1.total_up = long(pow(2, 62)) self.block1.total_down = long(pow(2, 62)) # Act self.db.add_block(self.block1) # Assert result = self.db.get_latest(self.block1.public_key) self.assertEqual_block(self.block1, result) @blocking_call_on_reactor_thread def test_get_insert_time(self): # Arrange # Upon adding the block to the database, the timestamp will get added. self.db.add_block(self.block1) # Act # Retrieving the block from the database will result in a block with a timestamp result = self.db.get_latest(self.block1.public_key) insert_time = datetime.datetime.strptime(result.insert_time, "%Y-%m-%d %H:%M:%S") # We store UTC timestamp time_difference = datetime.datetime.utcnow() - insert_time # Assert self.assertEquals(time_difference.days, 0) self.assertLess( time_difference.seconds, 10, "Difference in stored and retrieved time is too large.") @blocking_call_on_reactor_thread def set_db_version(self, version): self.db.executescript( u"UPDATE option SET value = '%d' WHERE key = 'database_version';" % version) self.db.close(commit=True) self.db = MultiChainDB(self.getStateDir()) @blocking_call_on_reactor_thread def test_database_upgrade(self): self.set_db_version(1) version, = next( self.db.execute( u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1" )) self.assertEqual(version, u"3") @blocking_call_on_reactor_thread def test_database_create(self): self.set_db_version(0) version, = next( self.db.execute( u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1" )) self.assertEqual(version, u"3") @blocking_call_on_reactor_thread def test_database_no_downgrade(self): self.set_db_version(200000) version, = next( self.db.execute( u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1" )) self.assertEqual(version, u"200000") @blocking_call_on_reactor_thread def test_block_to_dictionary(self): """ Test whether a block is correctly represented when converted to a dictionary """ block_dict = dict(self.block1) self.assertEqual(block_dict["up"], self.block1.up) self.assertEqual(block_dict["down"], self.block1.down) self.assertEqual(block_dict["insert_time"], self.block1.insert_time)
class TestDatabase(MultiChainTestCase): """ Tests the Database for MultiChain community. Also tests integration with Dispersy. This integration slows down the tests, but can probably be removed and a Mock Dispersy could be used. """ def __init__(self, *args, **kwargs): super(TestDatabase, self).__init__(*args, **kwargs) @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self, **kwargs): yield super(TestDatabase, self).setUp() path = os.path.join(self.getStateDir(), DATABASE_DIRECTORY) if not os.path.exists(path): os.makedirs(path) self.db = MultiChainDB(None, self.getStateDir()) self.block1 = TestBlock() self.block2 = TestBlock() @blocking_call_on_reactor_thread def test_add_block(self): # Act self.db.add_block(self.block1) # Assert result = self.db.get_by_hash_requester(self.block1.hash_requester) self.assertEqual_block(self.block1, result) @blocking_call_on_reactor_thread def test_get_blocks(self): """ Test whether the right blocks are returned when fetching blocks """ self.db.add_block(self.block1) self.assertEqual( len(self.db.get_blocks(self.block1.public_key_requester)), 1) self.block1.hash_requester = "a" * 40 self.db.add_block(self.block1) self.assertEqual( len(self.db.get_blocks(self.block1.public_key_requester)), 2) self.assertEqual( len(self.db.get_blocks(self.block1.public_key_requester, limit=1)), 1) @blocking_call_on_reactor_thread def test_get_num_interactors(self): """ Test whether the right number of interactors is returned """ crypto = ECCrypto() my_key = crypto.key_to_bin(crypto.generate_key(u"curve25519").pub()) block1 = TestBlock() block1.public_key_requester = my_key block1.up = 100 block1.down = 100 self.db.add_block(block1) block2 = TestBlock() block2.public_key_responder = my_key block2.up = 100 block2.down = 100 self.db.add_block(block2) self.assertEqual((2, 2), self.db.get_num_unique_interactors(my_key)) @blocking_call_on_reactor_thread def test_get_by_hash(self): # Act self.db.add_block(self.block1) # Assert result1 = self.db.get_by_hash_requester(self.block1.hash_requester) result2 = self.db.get_by_hash(self.block1.hash_requester) result3 = self.db.get_by_hash(self.block1.hash_responder) self.assertEqual_block(self.block1, result1) self.assertEqual_block(self.block1, result2) self.assertEqual_block(self.block1, result3) @blocking_call_on_reactor_thread def test_add_two_blocks(self): # Act self.db.add_block(self.block1) self.db.add_block(self.block2) # Assert result = self.db.get_by_hash_requester(self.block2.hash_requester) self.assertEqual_block(self.block2, result) @blocking_call_on_reactor_thread def test_get_block_non_existing(self): # Act result = self.db.get_by_hash_requester(self.block1.hash_requester) # Assert self.assertEqual(None, result) @blocking_call_on_reactor_thread def test_contains_block_id_positive(self): # Act self.db.add_block(self.block1) # Assert self.assertTrue(self.db.contains(self.block1.hash_requester)) @blocking_call_on_reactor_thread def test_contains_block_id_negative(self): # Act & Assert self.assertFalse(self.db.contains("NON EXISTING ID")) @blocking_call_on_reactor_thread def test_get_latest_sequence_number_not_existing(self): # Act & Assert self.assertEquals( self.db.get_latest_sequence_number("NON EXISTING KEY"), -1) @blocking_call_on_reactor_thread def test_get_latest_sequence_number_public_key_requester(self): # Arrange # Make sure that there is a responder block with a lower sequence number. # To test that it will look for both responder and requester. self.db.add_block(self.block1) self.block2.public_key_responder = self.block1.public_key_requester self.block2.sequence_number_responder = self.block1.sequence_number_requester - 5 self.db.add_block(self.block2) # Act & Assert self.assertEquals( self.db.get_latest_sequence_number( self.block1.public_key_requester), self.block1.sequence_number_requester) @blocking_call_on_reactor_thread def test_get_latest_sequence_number_public_key_responder(self): # Arrange # Make sure that there is a requester block with a lower sequence number. # To test that it will look for both responder and requester. self.db.add_block(self.block1) self.block2.public_key_requester = self.block1.public_key_responder self.block2.sequence_number_requester = self.block1.sequence_number_responder - 5 self.db.add_block(self.block2) # Act & Assert self.assertEquals( self.db.get_latest_sequence_number( self.block1.public_key_responder), self.block1.sequence_number_responder) @blocking_call_on_reactor_thread def test_get_previous_id_not_existing(self): # Act & Assert self.assertEquals(self.db.get_latest_hash("NON EXISTING KEY"), None) @blocking_call_on_reactor_thread def test_get_previous_hash_of_requester(self): # Arrange # Make sure that there is a responder block with a lower sequence number. # To test that it will look for both responder and requester. self.db.add_block(self.block1) self.block2.public_key_responder = self.block1.public_key_requester self.block2.sequence_number_responder = self.block1.sequence_number_requester + 1 self.db.add_block(self.block2) # Act & Assert self.assertEquals( self.db.get_latest_hash(self.block2.public_key_responder), self.block2.hash_responder) @blocking_call_on_reactor_thread def test_get_previous_hash_of_responder(self): # Arrange # Make sure that there is a requester block with a lower sequence number. # To test that it will look for both responder and requester. self.db.add_block(self.block1) self.block2.public_key_requester = self.block1.public_key_responder self.block2.sequence_number_requester = self.block1.sequence_number_responder + 1 self.db.add_block(self.block2) # Act & Assert self.assertEquals( self.db.get_latest_hash(self.block2.public_key_requester), self.block2.hash_requester) @blocking_call_on_reactor_thread def test_get_by_sequence_number_by_mid_not_existing(self): # Act & Assert self.assertEquals( self.db.get_by_public_key_and_sequence_number( "NON EXISTING KEY", 0), None) @blocking_call_on_reactor_thread def test_get_by_public_key_and_sequence_number_requester(self): # Arrange # Make sure that there is a responder block with a lower sequence number. # To test that it will look for both responder and requester. self.db.add_block(self.block1) # Act & Assert self.assertEqual_block( self.block1, self.db.get_by_public_key_and_sequence_number( self.block1.public_key_requester, self.block1.sequence_number_requester)) @blocking_call_on_reactor_thread def test_get_by_public_key_and_sequence_number_responder(self): # Arrange # Make sure that there is a responder block with a lower sequence number. # To test that it will look for both responder and requester. self.db.add_block(self.block1) # Act & Assert self.assertEqual_block( self.block1, self.db.get_by_public_key_and_sequence_number( self.block1.public_key_responder, self.block1.sequence_number_responder)) @blocking_call_on_reactor_thread def test_get_total(self): # Arrange self.db.add_block(self.block1) self.block2.public_key_requester = self.block1.public_key_responder self.block2.sequence_number_requester = self.block1.sequence_number_responder + 1 self.block2.total_up_requester = self.block1.total_up_responder + self.block2.up self.block2.total_down_requester = self.block1.total_down_responder + self.block2.down self.db.add_block(self.block2) # Act (result_up, result_down) = self.db.get_total(self.block2.public_key_requester) # Assert self.assertEqual(self.block2.total_up_requester, result_up) self.assertEqual(self.block2.total_down_requester, result_down) @blocking_call_on_reactor_thread def test_get_total_not_existing(self): # Arrange self.db.add_block(self.block1) # Act (result_up, result_down) = self.db.get_total(self.block2.public_key_requester) # Assert self.assertEqual(0, result_up) self.assertEqual(0, result_down) @blocking_call_on_reactor_thread def test_save_large_upload_download_block(self): """ Test if the block can save very large numbers. """ # Arrange self.block1.total_up_requester = long(pow(2, 62)) self.block1.total_down_requester = long(pow(2, 62)) self.block1.total_up_responder = long(pow(2, 61)) self.block1.total_down_responder = pow(2, 60) # Act self.db.add_block(self.block1) # Assert result = self.db.get_by_hash(self.block1.hash_requester) self.assertEqual_block(self.block1, result) @blocking_call_on_reactor_thread def test_get_insert_time(self): # Arrange # Upon adding the block to the database, the timestamp will get added. self.db.add_block(self.block1) # Act # Retrieving the block from the database will result in a block with a # timestamp result = self.db.get_by_hash(self.block1.hash_requester) insert_time = datetime.datetime.strptime(result.insert_time, "%Y-%m-%d %H:%M:%S") # We store UTC timestamp time_difference = datetime.datetime.utcnow() - insert_time # Assert self.assertEquals(time_difference.days, 0) self.assertLess( time_difference.seconds, 10, "Difference in stored and retrieved time is too large.") @blocking_call_on_reactor_thread def set_db_version(self, version): self.db.executescript( u"UPDATE option SET value = '%d' WHERE key = 'database_version';" % version) self.db.close(commit=True) self.db = MultiChainDB(None, self.getStateDir()) @blocking_call_on_reactor_thread def test_database_upgrade(self): self.set_db_version(1) version, = next( self.db.execute( u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1" )) self.assertEqual(version, u"2") @blocking_call_on_reactor_thread def test_database_create(self): self.set_db_version(0) version, = next( self.db.execute( u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1" )) self.assertEqual(version, u"2") @blocking_call_on_reactor_thread def test_database_no_downgrade(self): self.set_db_version(200000) version, = next( self.db.execute( u"SELECT value FROM option WHERE key = 'database_version' LIMIT 1" )) self.assertEqual(version, u"200000") @blocking_call_on_reactor_thread def test_block_to_dictionary(self): """ Test whether a block is correctly represented when converted to a dictionary """ block_dict = self.block1.to_dictionary() self.assertEqual(block_dict["up"], self.block1.up) self.assertEqual(block_dict["down"], self.block1.down) self.assertEqual(block_dict["insert_time"], self.block1.insert_time)
class MultiChainCommunity(Community): """ Community for reputation based on MultiChain tamper proof interaction history. """ def __init__(self, *args, **kwargs): super(MultiChainCommunity, self).__init__(*args, **kwargs) self.logger = logging.getLogger(self.__class__.__name__) self.notifier = None self.persistence = MultiChainDB(self.dispersy.working_directory) # We store the bytes send and received in the tunnel community in a dictionary. # The key is the public key of the peer being interacted with, the value a tuple of the up and down bytes # This data is not used to create outgoing requests, but _only_ to verify incoming requests self.pending_bytes = dict() self.logger.debug( "The multichain community started with Public Key: %s", self.my_member.public_key.encode("hex")) def initialize(self, tribler_session=None): super(MultiChainCommunity, self).initialize() if tribler_session: self.notifier = tribler_session.notifier self.notifier.add_observer(self.on_tunnel_remove, NTFY_TUNNEL, [NTFY_REMOVE]) @classmethod def get_master_members(cls, dispersy): # generated: Sun Apr 23 10:06:29 2017 # curve: None # len: 571 bits ~ 144 bytes signature # pub: 170 3081a7301006072a8648ce3d020106052b8104002703819200040503dac58c19267f12cb0cf667e480816cd2574acae5293b5 # 9d7c3da32e02b4747f7e2e9e9c880d2e5e2ba8b7fcc9892cb39b797ef98483ffd58739ed20990f8e3df7d1ec5a7ad2c0338dc206c4383a # 943e3e2c682ac4b585880929a947ffd50057b575fc30ec88eada3ce6484e5e4d6fdf41984cd1e51aaacc5f9a51bcc8393aea1f786fc47c # bf994cb1339f706df4a # pub-sha1 b78a5e252bf2f7be8716c383734f325b9aaff844 # -----BEGIN PUBLIC KEY----- # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQFA9rFjBkmfxLLDPZn5ICBbNJXSsrl # KTtZ18PaMuArR0f34unpyIDS5eK6i3/MmJLLObeX75hIP/1Yc57SCZD44999HsWn # rSwDONwgbEODqUPj4saCrEtYWICSmpR//VAFe1dfww7Ijq2jzmSE5eTW/fQZhM0e # UaqsxfmlG8yDk66h94b8R8v5lMsTOfcG30o= # -----END PUBLIC KEY----- master_key = "3081a7301006072a8648ce3d020106052b8104002703819200040503dac58c19267f12cb0cf667e480816cd2574acae" \ "5293b59d7c3da32e02b4747f7e2e9e9c880d2e5e2ba8b7fcc9892cb39b797ef98483ffd58739ed20990f8e3df7d1ec5" \ "a7ad2c0338dc206c4383a943e3e2c682ac4b585880929a947ffd50057b575fc30ec88eada3ce6484e5e4d6fdf41984c" \ "d1e51aaacc5f9a51bcc8393aea1f786fc47cbf994cb1339f706df4a" return [dispersy.get_member(public_key=master_key.decode("HEX"))] def initiate_meta_messages(self): """ Setup all message that can be received by this community and the super classes. :return: list of meta messages. """ return super(MultiChainCommunity, self).initiate_meta_messages() + [ Message(self, HALF_BLOCK, NoAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), HalfBlockPayload(), self._generic_timeline_check, self.received_half_block), Message(self, CRAWL, MemberAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), CrawlRequestPayload(), self._generic_timeline_check, self.received_crawl_request) ] def initiate_conversions(self): return [DefaultConversion(self), MultiChainConversion(self)] def send_block(self, candidate, block): if candidate.get_member(): self.logger.debug( "Sending block to %s (%s)", candidate.get_member().public_key.encode("hex")[-8:], block) else: self.logger.debug("Sending block to %s (%s)", candidate, block) message = self.get_meta_message(HALF_BLOCK).impl( authentication=tuple(), distribution=(self.claim_global_time(), ), destination=(candidate, ), payload=(block, )) try: self.dispersy.store_update_forward([message], False, False, True) except DelayPacketByMissingMember: self.logger.warn( "Missing member in MultiChain community to send signature request to" ) def sign_block(self, candidate, public_key=None, bytes_up=None, bytes_down=None, linked=None): """ Create, sign, persist and send a block signed message :param candidate: The peer with whom you have interacted, as a dispersy candidate :param bytes_up: The bytes you have uploaded to the peer in this interaction :param bytes_down: The bytes you have downloaded from the peer in this interaction :param linked: The block that the requester is asking us to sign """ # NOTE to the future: This method reads from the database, increments and then writes back. If in some future # this method is allowed to execute in parallel, be sure to lock from before .create up to after .add_block assert bytes_up is None and bytes_down is None and linked is not None or \ bytes_up is not None and bytes_down is not None and linked is None, \ "Either provide a linked block or byte counts, not both" assert linked is None or linked.link_public_key == self.my_member.public_key, \ "Cannot counter sign block not addressed to self" assert linked is None or linked.link_sequence_number == UNKNOWN_SEQ, \ "Cannot counter sign block that is not a request" block = MultiChainBlock.create(self.persistence, self.my_member.public_key, linked) if linked is None: block.up = bytes_up block.down = bytes_down block.total_up += bytes_up block.total_down += bytes_down block.link_public_key = public_key block.sign(self.my_member.private_key) validation = block.validate(self.persistence) self.logger.info("Signed block to %s (%s) validation result %s", block.link_public_key.encode("hex")[-8:], block, validation) if validation[0] != ValidationResult.partial_next and validation[ 0] != ValidationResult.valid: self.logger.error("Signed block did not validate?! Result %s", repr(validation)) else: self.persistence.add_block(block) self.send_block(candidate, block) def received_half_block(self, messages): """ We've received a half block, either because we sent a SIGNED message to some one or we are crawling :param messages The half block messages """ self.logger.debug("Received %d half block messages.", len(messages)) for message in messages: blk = message.payload.block validation = blk.validate(self.persistence) self.logger.debug("Block validation result %s, %s, (%s)", validation[0], validation[1], blk) if validation[0] == ValidationResult.invalid: continue elif not self.persistence.contains(blk): self.persistence.add_block(blk) else: self.logger.debug("Received already known block (%s)", blk) # Is this a request, addressed to us, and have we not signed it already? if blk.link_sequence_number != UNKNOWN_SEQ or \ blk.link_public_key != self.my_member.public_key or \ self.persistence.get_linked(blk) is not None: continue self.logger.info("Received request block addressed to us (%s)", blk) # determine if we want to sign (i.e. the requesting public key has enough pending bytes) pend = self.pending_bytes.get(blk.public_key) if not pend or not pend.add(-blk.down, -blk.up): self.logger.info( "Request block counter party does not have enough bytes pending." ) continue crawl_task = "crawl_%s" % blk.hash # It is important that the request matches up with its previous block, gaps cannot be tolerated at # this point. We already dropped invalids, so here we delay this message if the result is partial, # partial_previous or no-info. We send a crawl request to the requester to (hopefully) close the gap if validation[0] == ValidationResult.partial_previous or validation[0] == ValidationResult.partial or \ validation[0] == ValidationResult.no_info: self.logger.info( "Request block could not be validated sufficiently, crawling requester. %s", validation) # Note that this code does not cover the scenario where we obtain this block indirectly. # We modified the counters to get here, correct pending bytes since we did not really sign the block yet pend.add(blk.down, blk.up) # Are we already waiting for this crawl to happen? # For example: it's taking longer than 5 secs or the block message reached us twice via different paths if self.is_pending_task_active(crawl_task): continue self.send_crawl_request( message.candidate, blk.public_key, max(GENESIS_SEQ, blk.sequence_number - 5)) # Make sure we get called again after a while. Note that the cleanup task on pend will prevent # us from waiting on the peer forever. self.register_task( crawl_task, reactor.callLater(5.0, self.received_half_block, [message])) else: self.sign_block(message.candidate, linked=blk) if self.is_pending_task_active(crawl_task): self.cancel_pending_task(crawl_task) continue def send_crawl_request(self, candidate, public_key, sequence_number=None): sq = sequence_number if sequence_number is None: blk = self.persistence.get_latest(public_key) sq = blk.sequence_number if blk else GENESIS_SEQ sq = max(GENESIS_SEQ, sq) self.logger.info("Requesting crawl of node %s:%d", public_key.encode("hex")[-8:], sq) message = self.get_meta_message(CRAWL).impl( authentication=(self.my_member, ), distribution=(self.claim_global_time(), ), destination=(candidate, ), payload=(sq, )) self.dispersy.store_update_forward([message], False, False, True) def received_crawl_request(self, messages): self.logger.debug("Received %d crawl messages.", len(messages)) for message in messages: self.logger.info( "Received crawl request from node %s for sequence number %d", message.candidate.get_member().public_key.encode("hex")[-8:], message.payload.requested_sequence_number) blocks = self.persistence.crawl( self.my_member.public_key, message.payload.requested_sequence_number) count = len(blocks) for blk in blocks: self.send_block(message.candidate, blk) self.logger.info("Sent %d blocks", count) @blocking_call_on_reactor_thread def get_statistics(self, public_key=None): """ Returns a dictionary with some statistics regarding the local multichain database :returns a dictionary with statistics """ if public_key is None: public_key = self.my_member.public_key latest_block = self.persistence.get_latest(public_key) statistics = dict() statistics["id"] = public_key.encode("hex") interacts = self.persistence.get_num_unique_interactors(public_key) statistics["peers_that_pk_helped"] = interacts[0] if interacts[ 0] is not None else 0 statistics["peers_that_helped_pk"] = interacts[1] if interacts[ 1] is not None else 0 if latest_block: statistics["total_blocks"] = latest_block.sequence_number statistics["total_up"] = latest_block.total_up statistics["total_down"] = latest_block.total_down statistics["latest_block"] = dict(latest_block) else: statistics["total_blocks"] = 0 statistics["total_up"] = 0 statistics["total_down"] = 0 return statistics @inlineCallbacks def unload_community(self): self.logger.debug("Unloading the MultiChain Community.") if self.notifier: self.notifier.remove_observer(self.on_tunnel_remove) for pk in self.pending_bytes: if self.pending_bytes[pk].clean is not None: self.pending_bytes[pk].clean.reset(0) yield super(MultiChainCommunity, self).unload_community() # Close the persistence layer self.persistence.close() @forceDBThread def on_tunnel_remove(self, subject, change_type, tunnel, candidate): """ Handler for the remove event of a tunnel. This function will attempt to create a block for the amounts that were transferred using the tunnel. :param subject: Category of the notifier event :param change_type: Type of the notifier event :param tunnel: The tunnel that was removed (closed) :param candidate: The dispersy candidate with whom this node has interacted in the tunnel """ from Tribler.community.tunnel.tunnel_community import Circuit, RelayRoute, TunnelExitSocket assert isinstance(tunnel, Circuit) or isinstance(tunnel, RelayRoute) or isinstance(tunnel, TunnelExitSocket), \ "on_tunnel_remove() was called with an object that is not a Circuit, RelayRoute or TunnelExitSocket" assert isinstance(tunnel.bytes_up, int) and isinstance(tunnel.bytes_down, int),\ "tunnel instance must provide byte counts in int" up = tunnel.bytes_up down = tunnel.bytes_down pk = candidate.get_member().public_key # If the transaction is not big enough we discard the bytes up and down. if up + down >= MIN_TRANSACTION_SIZE: # Tie breaker to prevent both parties from requesting if up > down or (up == down and self.my_member.public_key > pk): self.register_task( "sign_%s" % tunnel.circuit_id, reactor.callLater(5, self.sign_block, candidate, pk, tunnel.bytes_up, tunnel.bytes_down)) else: pend = self.pending_bytes.get(pk) if not pend: self.pending_bytes[pk] = PendingBytes( up, down, reactor.callLater(2 * 60, self.cleanup_pending, pk)) else: pend.add(up, down) def cleanup_pending(self, public_key): self.pending_bytes.pop(public_key, None)
class MultiChainCommunity(Community): """ Community for reputation based on MultiChain tamper proof interaction history. """ def __init__(self, *args, **kwargs): super(MultiChainCommunity, self).__init__(*args, **kwargs) self.logger = logging.getLogger(self.__class__.__name__) self.notifier = None self._private_key = self.my_member.private_key self._public_key = self.my_member.public_key self.persistence = MultiChainDB(self.dispersy, self.dispersy.working_directory) self.logger.debug( "The multichain community started with Public Key: %s", base64.encodestring(self._public_key)) # No response is expected yet. self.expected_response = None def initialize(self, tribler_session=None): super(MultiChainCommunity, self).initialize() if tribler_session: self.notifier = tribler_session.notifier self.notifier.add_observer(self.on_tunnel_remove, NTFY_TUNNEL, [NTFY_REMOVE]) @classmethod def get_master_members(cls, dispersy): # generated: Fri Jul 1 15:22:20 2016 # curve: None # len:571 bits ~ 144 bytes signature # pub: 170 3081a7301006072a8648ce3d020106052b81040027038192000407afa96c83660dccfbf02a45b68f4bc # 4957539860a3fe1ad4a18ccbfc2a60af1174e1f5395a7917285d09ab67c3d80c56caf5396fc5b231d84ceac23627 # 930b4c35cbfce63a49805030dabbe9b5302a966b80eefd7003a0567c65ccec5ecde46520cfe1875b1187d469823d # 221417684093f63c33a8ff656331898e4bc853bcfaac49bc0b2a99028195b7c7dca0aea65 # pub-sha1 15ade4f5fb0f0f8019d8430473aaba4305e61753 # -----BEGIN PUBLIC KEY----- # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQHr6lsg2YNzPvwKkW2j0vElXU5hgo/4a1KGMy/wqYK8RdOH1OVp5FyhdCatnw9g # MVsr1OW/FsjHYTOrCNieTC0w1y/zmOkmAUDDau+m1MCqWa4Du/XADoFZ8ZczsXs3kZSDP4YdbEYfUaYI9IhQXaECT9jwzqP9l # YzGJjkvIU7z6rEm8CyqZAoGVt8fcoK6mU= # -----END PUBLIC KEY----- master_key = "3081a7301006072a8648ce3d020106052b81040027038192000407afa96c83660dccfbf02a45b68f4bc" + \ "4957539860a3fe1ad4a18ccbfc2a60af1174e1f5395a7917285d09ab67c3d80c56caf5396fc5b231d84ceac23627" + \ "930b4c35cbfce63a49805030dabbe9b5302a966b80eefd7003a0567c65ccec5ecde46520cfe1875b1187d469823d" + \ "221417684093f63c33a8ff656331898e4bc853bcfaac49bc0b2a99028195b7c7dca0aea65" master_key_hex = master_key.decode("HEX") master = dispersy.get_member(public_key=master_key_hex) return [master] def initiate_meta_messages(self): """ Setup all message that can be received by this community and the super classes. :return: list of meta messages. """ return super(MultiChainCommunity, self).initiate_meta_messages() + [ Message( self, SIGNATURE, DoubleMemberAuthentication( allow_signature_func=self.allow_signature_request, split_payload_func=split_function), PublicResolution(), DirectDistribution(), CandidateDestination(), SignaturePayload(), self._generic_timeline_check, self.received_signature_response), Message(self, CRAWL_REQUEST, MemberAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), CrawlRequestPayload(), self._generic_timeline_check, self.received_crawl_request), Message(self, CRAWL_RESPONSE, MemberAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), CrawlResponsePayload(), self._generic_timeline_check, self.received_crawl_response), Message(self, CRAWL_RESUME, MemberAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), CrawlResumePayload(), self._generic_timeline_check, self.received_crawl_resumption) ] def initiate_conversions(self): return [DefaultConversion(self), MultiChainConversion(self)] def schedule_block(self, candidate, bytes_up, bytes_down): """ Schedule a block for the current outstanding amounts :param candidate: The peer with whom you have interacted, as a dispersy candidate :param bytes_up: The bytes you have uploaded to the peer in this interaction :param bytes_down: The bytes you have downloaded from the peer in this interaction """ self.logger.info("MULTICHAIN: Schedule Block called. Candidate: " + str(candidate) + " UP: " + str(bytes_up) + " DOWN: " + str(bytes_down)) self.add_discovered_candidate(candidate) if candidate and candidate.get_member(): # Convert to MB total_amount_sent_mb = bytes_up / MEGA_DIVIDER total_amount_received_mb = bytes_down / MEGA_DIVIDER # Try to send the request try: self.publish_signature_request_message( candidate, total_amount_sent_mb, total_amount_received_mb) except DelayPacketByMissingMember: self.logger.warn( "Missing member in MultiChain community to send signature request to" ) else: self.logger.warn( "No valid candidate found for: %s to request block from.", candidate) def publish_signature_request_message(self, candidate, up, down): """ Creates and sends out a signed signature_request message Returns true upon success :param candidate: The candidate that the signature request will be sent to. :param (int) up: The amount of Megabytes that have been sent to the candidate that need to signed. :param (int) down: The amount of Megabytes that have been received from the candidate that need to signed. return (bool) if request is sent. """ message = self.create_signature_request_message(candidate, up, down) self.create_signature_request(candidate, message, self.allow_signature_response) self.persist_signature_request(message) return True def create_signature_request_message(self, candidate, up, down): """ Create a signature request message using the current time stamp. :param candidate: The candidate that the signature request will be sent to. :param (int) up: The amount of Megabytes that have been sent to the candidate that need to signed. :param (int) down: The amount of Megabytes that have been received from the candidate that need to signed. :return: Signature_request message ready for distribution. """ # Instantiate the data total_up_requester, total_down_requester = self._get_next_total( up, down) # Instantiate the personal information sequence_number_requester = self._get_next_sequence_number() previous_hash_requester = self._get_latest_hash() payload = (up, down, total_up_requester, total_down_requester, sequence_number_requester, previous_hash_requester) meta = self.get_meta_message(SIGNATURE) message = meta.impl( authentication=([self.my_member, candidate.get_member()], ), distribution=(self.claim_global_time(), ), payload=payload) return message def allow_signature_request(self, message): """ We've received a signature request message, we must either: a. Create and sign the response part of the message, send it back, and persist the block. b. Drop the message. (Future work: notify the sender of dropping) :param message The message containing the received signature request. """ self.logger.info("Received signature request for: [Up = " + str(message.payload.up) + "MB | Down = " + str(message.payload.down) + " MB]") # TODO: This code always signs a request. Checks and rejects should be inserted here! # TODO: Like basic total_up == previous_total_up + block.up or more sophisticated chain checks. payload = message.payload # The up and down values are reversed for the responder. total_up_responder, total_down_responder = self._get_next_total( payload.down, payload.up) sequence_number_responder = self._get_next_sequence_number() previous_hash_responder = self._get_latest_hash() payload = (payload.up, payload.down, payload.total_up_requester, payload.total_down_requester, payload.sequence_number_requester, payload.previous_hash_requester, total_up_responder, total_down_responder, sequence_number_responder, previous_hash_responder) meta = self.get_meta_message(SIGNATURE) message = meta.impl(authentication=(message.authentication.members, message.authentication.signatures), distribution=(message.distribution.global_time, ), payload=payload) self.persist_signature_response(message) self.logger.info("Sending signature response.") return message def allow_signature_response(self, request, response, modified): """ We've received a signature response message after sending a request, we must return either: a. True, if we accept this message b. False, if not (because of inconsistencies in the payload) :param request The original message as send by this node :param response The response message received :param modified (bool) True if the message was modified """ if not response: self.logger.info("Timeout received for signature request.") return False else: # TODO: Check whether we are expecting a response self.logger.info("Signature response received. Modified: %s", modified) return (request.payload.sequence_number_requester == response.payload.sequence_number_requester and request.payload.previous_hash_requester == response.payload.previous_hash_requester and modified) def received_signature_response(self, messages): """ We've received a valid signature response and must process this message. :param messages The received, and validated signature response messages """ self.logger.info("Valid %s signature response(s) received.", len(messages)) for message in messages: self.update_signature_response(message) def persist_signature_response(self, message): """ Persist the signature response message, when this node has not yet persisted the corresponding request block. A hash will be created from the message and this will be used as an unique identifier. :param message: """ block = DatabaseBlock.from_signature_response_message(message) self.logger.info("Persisting sr: %s", base64.encodestring(block.hash_requester).strip()) self.persistence.add_block(block) def update_signature_response(self, message): """ Update the signature response message, when this node has already persisted the corresponding request block. A hash will be created from the message and this will be used as an unique identifier. :param message: """ block = DatabaseBlock.from_signature_response_message(message) self.logger.info("Persisting sr: %s", base64.encodestring(block.hash_requester).strip()) self.persistence.update_block_with_responder(block) def persist_signature_request(self, message): """ Persist the signature request message as a block. The block will be updated when a response is received. :param message: """ block = DatabaseBlock.from_signature_request_message(message) self.logger.info("Persisting sr: %s", base64.encodestring(block.hash_requester).strip()) self.persistence.add_block(block) def send_crawl_request(self, candidate, sequence_number=None): if sequence_number is None: sequence_number = self.persistence.get_latest_sequence_number( candidate.get_member().public_key) self.logger.info( "Crawler: Requesting crawl from node %s, from sequence number %d", base64.encodestring(candidate.get_member().mid).strip(), sequence_number) meta = self.get_meta_message(CRAWL_REQUEST) message = meta.impl(authentication=(self.my_member, ), distribution=(self.claim_global_time(), ), destination=(candidate, ), payload=(sequence_number, )) self.dispersy.store_update_forward([message], False, False, True) def received_crawl_request(self, messages): for message in messages: self.logger.info( "Crawler: Received crawl request from node %s, from sequence number %d", base64.encodestring( message.candidate.get_member().mid).strip(), message.payload.requested_sequence_number) self.crawl_requested(message.candidate, message.payload.requested_sequence_number) def crawl_requested(self, candidate, sequence_number): blocks = self.persistence.get_blocks_since(self._public_key, sequence_number) if len(blocks) > 0: self.logger.debug("Crawler: Sending %d blocks", len(blocks)) messages = [ self.get_meta_message(CRAWL_RESPONSE).impl( authentication=(self.my_member, ), distribution=(self.claim_global_time(), ), destination=(candidate, ), payload=block.to_payload()) for block in blocks ] self.dispersy.store_update_forward(messages, False, False, True) if len(blocks) > 1: # we sent more than 1 block. Send a resumption token so the other side knows it should continue crawling # last_block = blocks[-1] # resumption_number = last_block.sequence_number_requster if # last_block.mid_requester == self._mid else last_block.sequence_number_responder message = self.get_meta_message(CRAWL_RESUME).impl( authentication=(self.my_member, ), distribution=(self.claim_global_time(), ), destination=(candidate, ), # payload=(resumption_number)) payload=()) self.dispersy.store_update_forward([message], False, False, True) else: # This is slightly worrying since the last block should always be returned. # Or rather, the other side is requesting blocks starting from a point in the future. self.logger.info("Crawler: No blocks") def received_crawl_response(self, messages): self.logger.debug("Crawler: Valid %d block response(s) received.", len(messages)) for message in messages: requester = self.dispersy.get_member( public_key=message.payload.public_key_requester) responder = self.dispersy.get_member( public_key=message.payload.public_key_responder) block = DatabaseBlock.from_block_response_message( message, requester, responder) # Create the hash of the message if not self.persistence.contains(block.hash_requester): self.logger.info( "Crawler: Persisting sr: %s from ip (%s:%d)", base64.encodestring(block.hash_requester).strip(), message.candidate.sock_addr[0], message.candidate.sock_addr[1]) self.persistence.add_block(block) else: self.logger.debug("Crawler: Received already known block") def received_crawl_resumption(self, messages): self.logger.info("Crawler: Valid %s crawl resumptions received.", len(messages)) for message in messages: self.send_crawl_request(message.candidate) @blocking_call_on_reactor_thread def get_statistics(self): """ Returns a dictionary with some statistics regarding the local multichain database :returns a dictionary with statistics """ statistics = dict() statistics["self_id"] = base64.encodestring(self._public_key).strip() statistics[ "self_total_blocks"] = self.persistence.get_latest_sequence_number( self._public_key) (statistics["self_total_up_mb"], statistics["self_total_down_mb"]) = self.persistence.get_total( self._public_key) (statistics["self_peers_helped"], statistics["self_peers_helped_you"] ) = self.persistence.get_num_unique_interactors(self._public_key) latest_block = self.persistence.get_latest_block(self._public_key) if latest_block: statistics["latest_block_insert_time"] = str( latest_block.insert_time) statistics["latest_block_id"] = base64.encodestring( latest_block.hash_requester).strip() statistics["latest_block_requester_id"] = base64.encodestring( latest_block.public_key_requester).strip() statistics["latest_block_responder_id"] = base64.encodestring( latest_block.public_key_responder).strip() statistics["latest_block_up_mb"] = str(latest_block.up) statistics["latest_block_down_mb"] = str(latest_block.down) else: statistics["latest_block_insert_time"] = "" statistics["latest_block_id"] = "" statistics["latest_block_requester_id"] = "" statistics["latest_block_responder_id"] = "" statistics["latest_block_up_mb"] = "" statistics["latest_block_down_mb"] = "" return statistics def _get_next_total(self, up, down): """ Returns the next total numbers of up and down incremented with the current interaction up and down metric. :param up: Up metric for the interaction. :param down: Down metric for the interaction. :return: (total_up (int), total_down (int) """ total_up, total_down = self.persistence.get_total(self._public_key) return total_up + up, total_down + down def _get_next_sequence_number(self): return self.persistence.get_latest_sequence_number( self._public_key) + 1 def _get_latest_hash(self): previous_hash = self.persistence.get_latest_hash(self._public_key) return previous_hash if previous_hash else GENESIS_ID @inlineCallbacks def unload_community(self): self.logger.debug("Unloading the MultiChain Community.") if self.notifier: self.notifier.remove_observer(self.on_tunnel_remove) yield super(MultiChainCommunity, self).unload_community() # Close the persistence layer self.persistence.close() @forceDBThread def on_tunnel_remove(self, subject, change_type, tunnel, candidate): """ Handler for the remove event of a tunnel. This function will attempt to create a block for the amounts that were transferred using the tunnel. :param subject: Category of the notifier event :param change_type: Type of the notifier event :param tunnel: The tunnel that was removed (closed) :param candidate: The dispersy candidate with whom this node has interacted in the tunnel """ from Tribler.community.tunnel.tunnel_community import Circuit, RelayRoute, TunnelExitSocket assert isinstance(tunnel, Circuit) or isinstance(tunnel, RelayRoute) or isinstance(tunnel, TunnelExitSocket), \ "on_tunnel_remove() was called with an object that is not a Circuit, RelayRoute or TunnelExitSocket" if isinstance(tunnel.bytes_up, int) and isinstance( tunnel.bytes_down, int): if tunnel.bytes_up > MEGA_DIVIDER or tunnel.bytes_down > MEGA_DIVIDER: # Tie breaker to prevent both parties from requesting if self._public_key > candidate.get_member().public_key: self.schedule_block(candidate, tunnel.bytes_up, tunnel.bytes_down)