def _init_connection_manager(self, seek_head_blob_first=False): # this import is requierd here so utils.call_later is replaced by self.clock.callLater from lbrynet.core.client.ConnectionManager import ConnectionManager self.connection_manager = ConnectionManager(self.downloader, self.rate_limiter, [self.primary_request_creator], []) self.connection_manager.seek_head_blob_first = seek_head_blob_first self.connection_manager._start()
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager): log.debug("Try to download %s from %s", blob_hash, peer.host) blob_manager = blob_manager blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout) download_manager = SingleBlobDownloadManager(blob) peer_finder = SinglePeerFinder(peer) requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager, self._wallet, download_manager) downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder, self._rate_limiter, self._payment_rate_manager, self._wallet, timeout=timeout) info_exchanger = self._wallet.get_info_exchanger() connection_manager = ConnectionManager(downloader, self._rate_limiter, [requester], [info_exchanger]) connection_manager.start() result = yield blob.callback if not result: log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host) yield connection_manager.stop() defer.returnValue(result)
def download(self): if not is_valid_blobhash(self.blob_hash): return defer.fail(Failure(InvalidBlobHashError(self.blob_hash))) def cancel_download(d): self.stop() self.finished_deferred = defer.Deferred(canceller=cancel_download) self.download_manager = DownloadManager(self.blob_manager, True) self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder, self.payment_rate_manager, self.wallet, self.download_manager) self.download_manager.blob_info_finder = SingleBlobMetadataHandler(self.blob_hash, self.download_manager) self.download_manager.progress_manager = SingleProgressManager(self._blob_downloaded, self.download_manager) self.download_manager.blob_handler = DummyBlobHandler() self.download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger() self.download_manager.connection_manager = ConnectionManager( self, self.rate_limiter, [self.download_manager.blob_requester], [self.download_manager.wallet_info_exchanger] ) d = self.download_manager.start_downloading() d.addCallback(lambda _: self.finished_deferred) return d
def setUp(self): conf.initialize_settings() self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT) self.downloader = MocDownloader() self.rate_limiter = RateLimiter() self.primary_request_creator = MocRequestCreator([self.TEST_PEER]) self.clock = task.Clock() utils.call_later = self.clock.callLater from lbrynet.core.client.ConnectionManager import ConnectionManager self.connection_manager = ConnectionManager( self.downloader, self.rate_limiter, [self.primary_request_creator], []) self.connection_manager._start() self.server_port = None
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager): log.debug("Try to download %s from %s", blob_hash, peer.host) blob_manager = blob_manager blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout) download_manager = SingleBlobDownloadManager(blob) peer_finder = SinglePeerFinder(peer) requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager, self._wallet, download_manager) downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder, self._rate_limiter, self._payment_rate_manager, self._wallet, timeout=timeout) info_exchanger = self._wallet.get_info_exchanger() connection_manager = ConnectionManager(downloader, self._rate_limiter, [requester], [info_exchanger]) connection_manager.start() result = yield blob.callback if not result: log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host) yield connection_manager.stop() defer.returnValue(result)
def _start(self): self.download_manager = DownloadManager(self.blob_manager, True) info_finder = BlindMetadataHandler(self.info_manager, self.peers, self.peer_finder, self.approved_peers, self.payment_rate_manager, self.wallet, self.download_manager) self.download_manager.blob_info_finder = info_finder blob_requester = BlobRequester(self.blob_manager, self.peer_finder, self.payment_rate_manager, self.wallet, self.download_manager) self.download_manager.blob_requester = blob_requester self.progress_manager = BlindProgressManager(self.blob_manager, self.peers, self.max_space, [PeerScoreBasedScorer(), LengthBasedScorer()], self.download_manager) self.download_manager.progress_manager = self.progress_manager self.download_manager.blob_handler = BlindBlobHandler() wallet_info_exchanger = self.wallet.get_info_exchanger() self.download_manager.wallet_info_exchanger = wallet_info_exchanger connection_manager = ConnectionManager(self, self.rate_limiter, [info_finder, blob_requester], [wallet_info_exchanger]) self.download_manager.connection_manager = connection_manager d = defer.maybeDeferred(self.download_manager.start_downloading) d.addCallback(lambda _: self._update_status(stopped=False)) return d
class TestIntegrationConnectionManager(TestCase): skip = 'times out, needs to be refactored to work with py3' def setUp(self): conf.initialize_settings(False) self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT) self.downloader = MocDownloader() self.rate_limiter = RateLimiter() self.primary_request_creator = MocRequestCreator([self.TEST_PEER]) self.clock = task.Clock() utils.call_later = self.clock.callLater self.server_port = None def _init_connection_manager(self, seek_head_blob_first=False): # this import is required here so utils.call_later is replaced by self.clock.callLater from lbrynet.core.client.ConnectionManager import ConnectionManager self.connection_manager = ConnectionManager( self.downloader, self.rate_limiter, [self.primary_request_creator], []) self.connection_manager.seek_head_blob_first = seek_head_blob_first self.connection_manager._start() def tearDown(self): if self.server_port is not None: self.server_port.stopListening() self.connection_manager.stop() conf.settings = None @defer.inlineCallbacks def test_success(self): self._init_connection_manager() # test to see that if we setup a server, we get a connection self.server = MocServerProtocolFactory(self.clock) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(1, self.TEST_PEER.success_count) self.assertEqual(0, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_server_with_improper_reply(self): self._init_connection_manager() self.server = MocServerProtocolFactory(self.clock, is_good=False) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_non_existing_server(self): # Test to see that if we don't setup a server, we don't get a connection self._init_connection_manager() yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertFalse(connection_made) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_parallel_connections(self): # Test to see that we make two new connections at a manage call, # without it waiting for the connection to complete self._init_connection_manager() test_peer2 = Peer(LOCAL_HOST, PEER_PORT + 1) self.primary_request_creator.peers_to_return = [ self.TEST_PEER, test_peer2 ] yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(2, self.connection_manager.num_peer_connections()) self.assertIn(self.TEST_PEER, self.connection_manager._peer_connections) self.assertIn(test_peer2, self.connection_manager._peer_connections) deferred_conn_made_peer1 = self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred deferred_conn_made_peer1.addCallback( lambda conn_made: self.assertFalse(conn_made)) deferred_conn_made_peer2 = self.connection_manager._peer_connections[test_peer2].\ factory.connection_was_made_deferred deferred_conn_made_peer2.addCallback( lambda conn_made: self.assertFalse(conn_made)) yield deferred_conn_made_peer1 yield deferred_conn_made_peer2 self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) self.assertEqual(0, test_peer2.success_count) self.assertEqual(1, test_peer2.down_count) @defer.inlineCallbacks def test_stop(self): # test to see that when we call stop, the ConnectionManager waits for the # current manage call to finish, closes connections, # and removes scheduled manage calls self._init_connection_manager() self.connection_manager.manage(schedule_next_call=True) yield self.connection_manager.stop() self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertIsNone(self.connection_manager._next_manage_call) @defer.inlineCallbacks def test_closed_connection_when_server_is_slow(self): self._init_connection_manager() self.server = MocServerProtocolFactory(self.clock, has_moc_query_handler=True, is_delayed=True) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) # test header first seeks @defer.inlineCallbacks def test_no_peer_for_head_blob(self): # test that if we can't find blobs for the head blob, # it looks at the next unavailable and makes connection self._init_connection_manager(seek_head_blob_first=True) self.server = MocServerProtocolFactory(self.clock) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) self.primary_request_creator.peers_to_return_head_blob = [] self.primary_request_creator.peers_to_return = [self.TEST_PEER] yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(1, self.TEST_PEER.success_count) self.assertEqual(0, self.TEST_PEER.down_count)
def _get_connection_manager(self, download_manager): return ConnectionManager( self, self.rate_limiter, self._get_primary_request_creators(download_manager), self._get_secondary_request_creators(download_manager))
class TestIntegrationConnectionManager(unittest.TestCase): def setUp(self): conf.initialize_settings() self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT) self.downloader = MocDownloader() self.rate_limiter = RateLimiter() self.primary_request_creator = MocRequestCreator([self.TEST_PEER]) self.clock = task.Clock() utils.call_later = self.clock.callLater self.server_port = None def _init_connection_manager(self, seek_head_blob_first=False): # this import is requierd here so utils.call_later is replaced by self.clock.callLater from lbrynet.core.client.ConnectionManager import ConnectionManager self.connection_manager = ConnectionManager(self.downloader, self.rate_limiter, [self.primary_request_creator], []) self.connection_manager.seek_head_blob_first = seek_head_blob_first self.connection_manager._start() def tearDown(self): if self.server_port is not None: self.server_port.stopListening() self.connection_manager.stop() conf.settings = None @defer.inlineCallbacks def test_success(self): self._init_connection_manager() # test to see that if we setup a server, we get a connection self.server = MocServerProtocolFactory(self.clock) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(1, self.TEST_PEER.success_count) self.assertEqual(0, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_server_with_improper_reply(self): self._init_connection_manager() self.server = MocServerProtocolFactory(self.clock, is_good=False) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_non_existing_server(self): # Test to see that if we don't setup a server, we don't get a connection self._init_connection_manager() yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertFalse(connection_made) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_parallel_connections(self): # Test to see that we make two new connections at a manage call, # without it waiting for the connection to complete self._init_connection_manager() test_peer2 = Peer(LOCAL_HOST, PEER_PORT+1) self.primary_request_creator.peers_to_return = [self.TEST_PEER, test_peer2] yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(2, self.connection_manager.num_peer_connections()) self.assertIn(self.TEST_PEER, self.connection_manager._peer_connections) self.assertIn(test_peer2, self.connection_manager._peer_connections) deferred_conn_made_peer1 = self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred deferred_conn_made_peer1.addCallback(lambda conn_made: self.assertFalse(conn_made)) deferred_conn_made_peer2 = self.connection_manager._peer_connections[test_peer2].\ factory.connection_was_made_deferred deferred_conn_made_peer2.addCallback(lambda conn_made: self.assertFalse(conn_made)) yield deferred_conn_made_peer1 yield deferred_conn_made_peer2 self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) self.assertEqual(0, test_peer2.success_count) self.assertEqual(1, test_peer2.down_count) @defer.inlineCallbacks def test_stop(self): # test to see that when we call stop, the ConnectionManager waits for the # current manage call to finish, closes connections, # and removes scheduled manage calls self._init_connection_manager() self.connection_manager.manage(schedule_next_call=True) yield self.connection_manager.stop() self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(None, self.connection_manager._next_manage_call) @defer.inlineCallbacks def test_closed_connection_when_server_is_slow(self): self._init_connection_manager() self.server = MocServerProtocolFactory( self.clock, has_moc_query_handler=True, is_delayed=True) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(True, connection_made) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) # test header first seeks @defer.inlineCallbacks def test_no_peer_for_head_blob(self): # test that if we can't find blobs for the head blob, # it looks at the next unavailable and makes connection self._init_connection_manager(seek_head_blob_first=True) self.server = MocServerProtocolFactory(self.clock) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) self.primary_request_creator.peers_to_return_head_blob = [] self.primary_request_creator.peers_to_return = [self.TEST_PEER] yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\ factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(1, self.TEST_PEER.success_count) self.assertEqual(0, self.TEST_PEER.down_count)
class TestIntegrationConnectionManager(unittest.TestCase): def setUp(self): conf.initialize_settings() self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT) self.downloader = MocDownloader() self.rate_limiter = RateLimiter() self.primary_request_creator = MocRequestCreator([self.TEST_PEER]) self.clock = task.Clock() utils.call_later = self.clock.callLater from lbrynet.core.client.ConnectionManager import ConnectionManager self.connection_manager = ConnectionManager( self.downloader, self.rate_limiter, [self.primary_request_creator], []) self.connection_manager._start() self.server_port = None def tearDown(self): if self.server_port is not None: self.server_port.stopListening() self.connection_manager.stop() conf.settings = None @defer.inlineCallbacks def test_success(self): # test to see that if we setup a server, we get a connection self.server = MocServerProtocolFactory(self.clock) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[ self.TEST_PEER].factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(1, self.TEST_PEER.success_count) self.assertEqual(0, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_server_with_improper_reply(self): self.server = MocServerProtocolFactory(self.clock, is_good=False) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[ self.TEST_PEER].factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertTrue(connection_made) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) @defer.inlineCallbacks def test_non_existing_server(self): # Test to see that if we don't setup a server, we don't get a connection yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[ self.TEST_PEER].factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertFalse(connection_made) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) @unittest.SkipTest @defer.inlineCallbacks def test_parallel_connections(self): # Test to see that we make two new connections at a manage call, # without it waiting for the connection to complete test_peer2 = Peer(LOCAL_HOST, PEER_PORT + 1) self.primary_request_creator.peers_to_return = [ self.TEST_PEER, test_peer2 ] yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(2, self.connection_manager.num_peer_connections()) self.assertIn(self.TEST_PEER, self.connection_manager._peer_connections) self.assertIn(test_peer2, self.connection_manager._peer_connections) connection_made = yield self.connection_manager._peer_connections[ self.TEST_PEER].factory.connection_was_made_deferred self.assertFalse(connection_made) self.assertEqual(1, self.connection_manager.num_peer_connections()) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) connection_made = yield self.connection_manager._peer_connections[ test_peer2].factory.connection_was_made_deferred self.assertFalse(connection_made) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(0, test_peer2.success_count) self.assertEqual(1, test_peer2.down_count) @defer.inlineCallbacks def test_stop(self): # test to see that when we call stop, the ConnectionManager waits for the # current manage call to finish, closes connections, # and removes scheduled manage calls self.connection_manager.manage(schedule_next_call=True) yield self.connection_manager.stop() self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count) self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(None, self.connection_manager._next_manage_call) @defer.inlineCallbacks def test_closed_connection_when_server_is_slow(self): self.server = MocServerProtocolFactory(self.clock, has_moc_query_handler=True, is_delayed=True) self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST) yield self.connection_manager.manage(schedule_next_call=False) self.assertEqual(1, self.connection_manager.num_peer_connections()) connection_made = yield self.connection_manager._peer_connections[ self.TEST_PEER].factory.connection_was_made_deferred self.assertEqual(0, self.connection_manager.num_peer_connections()) self.assertEqual(True, connection_made) self.assertEqual(0, self.TEST_PEER.success_count) self.assertEqual(1, self.TEST_PEER.down_count)