def _init_connection_manager(self, seek_head_blob_first=False):
# this import is requierd here so utils.call_later is replaced by self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(self.downloader, self.rate_limiter,
[self.primary_request_creator], [])
self.connection_manager.seek_head_blob_first = seek_head_blob_first
self.connection_manager._start()
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager):
log.debug("Try to download %s from %s", blob_hash, peer.host)
blob_manager = blob_manager
blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout)
download_manager = SingleBlobDownloadManager(blob)
peer_finder = SinglePeerFinder(peer)
requester = BlobRequester(blob_manager, peer_finder,
self._payment_rate_manager, self._wallet,
download_manager)
downloader = StandaloneBlobDownloader(blob_hash,
blob_manager,
peer_finder,
self._rate_limiter,
self._payment_rate_manager,
self._wallet,
timeout=timeout)
info_exchanger = self._wallet.get_info_exchanger()
connection_manager = ConnectionManager(downloader, self._rate_limiter,
[requester], [info_exchanger])
connection_manager.start()
result = yield blob.callback
if not result:
log.debug("Failed to downloaded %s from %s", blob_hash[:16],
peer.host)
yield connection_manager.stop()
defer.returnValue(result)
def download(self):
if not is_valid_blobhash(self.blob_hash):
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
def cancel_download(d):
self.stop()
self.finished_deferred = defer.Deferred(canceller=cancel_download)
self.download_manager = DownloadManager(self.blob_manager, True)
self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder,
self.payment_rate_manager, self.wallet,
self.download_manager)
self.download_manager.blob_info_finder = SingleBlobMetadataHandler(self.blob_hash,
self.download_manager)
self.download_manager.progress_manager = SingleProgressManager(self._blob_downloaded,
self.download_manager)
self.download_manager.blob_handler = DummyBlobHandler()
self.download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
self.download_manager.connection_manager = ConnectionManager(
self, self.rate_limiter,
[self.download_manager.blob_requester],
[self.download_manager.wallet_info_exchanger]
)
d = self.download_manager.start_downloading()
d.addCallback(lambda _: self.finished_deferred)
return d
def setUp(self):
conf.initialize_settings()
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(
self.downloader, self.rate_limiter, [self.primary_request_creator],
[])
self.connection_manager._start()
self.server_port = None
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager):
log.debug("Try to download %s from %s", blob_hash, peer.host)
blob_manager = blob_manager
blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout)
download_manager = SingleBlobDownloadManager(blob)
peer_finder = SinglePeerFinder(peer)
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
self._wallet, download_manager)
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
self._rate_limiter, self._payment_rate_manager,
self._wallet, timeout=timeout)
info_exchanger = self._wallet.get_info_exchanger()
connection_manager = ConnectionManager(downloader, self._rate_limiter, [requester],
[info_exchanger])
connection_manager.start()
result = yield blob.callback
if not result:
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
yield connection_manager.stop()
defer.returnValue(result)
def _start(self):
self.download_manager = DownloadManager(self.blob_manager, True)
info_finder = BlindMetadataHandler(self.info_manager, self.peers, self.peer_finder,
self.approved_peers, self.payment_rate_manager,
self.wallet, self.download_manager)
self.download_manager.blob_info_finder = info_finder
blob_requester = BlobRequester(self.blob_manager, self.peer_finder, self.payment_rate_manager,
self.wallet, self.download_manager)
self.download_manager.blob_requester = blob_requester
self.progress_manager = BlindProgressManager(self.blob_manager, self.peers, self.max_space,
[PeerScoreBasedScorer(), LengthBasedScorer()],
self.download_manager)
self.download_manager.progress_manager = self.progress_manager
self.download_manager.blob_handler = BlindBlobHandler()
wallet_info_exchanger = self.wallet.get_info_exchanger()
self.download_manager.wallet_info_exchanger = wallet_info_exchanger
connection_manager = ConnectionManager(self, self.rate_limiter, [info_finder, blob_requester],
[wallet_info_exchanger])
self.download_manager.connection_manager = connection_manager
d = defer.maybeDeferred(self.download_manager.start_downloading)
d.addCallback(lambda _: self._update_status(stopped=False))
return d
class TestIntegrationConnectionManager(TestCase):
skip = 'times out, needs to be refactored to work with py3'
def setUp(self):
conf.initialize_settings(False)
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
self.server_port = None
def _init_connection_manager(self, seek_head_blob_first=False):
# this import is required here so utils.call_later is replaced by self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(
self.downloader, self.rate_limiter, [self.primary_request_creator],
[])
self.connection_manager.seek_head_blob_first = seek_head_blob_first
self.connection_manager._start()
def tearDown(self):
if self.server_port is not None:
self.server_port.stopListening()
self.connection_manager.stop()
conf.settings = None
@defer.inlineCallbacks
def test_success(self):
self._init_connection_manager()
# test to see that if we setup a server, we get a connection
self.server = MocServerProtocolFactory(self.clock)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(1, self.TEST_PEER.success_count)
self.assertEqual(0, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_server_with_improper_reply(self):
self._init_connection_manager()
self.server = MocServerProtocolFactory(self.clock, is_good=False)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_non_existing_server(self):
# Test to see that if we don't setup a server, we don't get a connection
self._init_connection_manager()
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertFalse(connection_made)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_parallel_connections(self):
# Test to see that we make two new connections at a manage call,
# without it waiting for the connection to complete
self._init_connection_manager()
test_peer2 = Peer(LOCAL_HOST, PEER_PORT + 1)
self.primary_request_creator.peers_to_return = [
self.TEST_PEER, test_peer2
]
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(2, self.connection_manager.num_peer_connections())
self.assertIn(self.TEST_PEER,
self.connection_manager._peer_connections)
self.assertIn(test_peer2, self.connection_manager._peer_connections)
deferred_conn_made_peer1 = self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
deferred_conn_made_peer1.addCallback(
lambda conn_made: self.assertFalse(conn_made))
deferred_conn_made_peer2 = self.connection_manager._peer_connections[test_peer2].\
factory.connection_was_made_deferred
deferred_conn_made_peer2.addCallback(
lambda conn_made: self.assertFalse(conn_made))
yield deferred_conn_made_peer1
yield deferred_conn_made_peer2
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
self.assertEqual(0, test_peer2.success_count)
self.assertEqual(1, test_peer2.down_count)
@defer.inlineCallbacks
def test_stop(self):
# test to see that when we call stop, the ConnectionManager waits for the
# current manage call to finish, closes connections,
# and removes scheduled manage calls
self._init_connection_manager()
self.connection_manager.manage(schedule_next_call=True)
yield self.connection_manager.stop()
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertIsNone(self.connection_manager._next_manage_call)
@defer.inlineCallbacks
def test_closed_connection_when_server_is_slow(self):
self._init_connection_manager()
self.server = MocServerProtocolFactory(self.clock,
has_moc_query_handler=True,
is_delayed=True)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
# test header first seeks
@defer.inlineCallbacks
def test_no_peer_for_head_blob(self):
# test that if we can't find blobs for the head blob,
# it looks at the next unavailable and makes connection
self._init_connection_manager(seek_head_blob_first=True)
self.server = MocServerProtocolFactory(self.clock)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
self.primary_request_creator.peers_to_return_head_blob = []
self.primary_request_creator.peers_to_return = [self.TEST_PEER]
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(1, self.TEST_PEER.success_count)
self.assertEqual(0, self.TEST_PEER.down_count)
def _get_connection_manager(self, download_manager):
return ConnectionManager(
self, self.rate_limiter,
self._get_primary_request_creators(download_manager),
self._get_secondary_request_creators(download_manager))
class TestIntegrationConnectionManager(unittest.TestCase):
def setUp(self):
conf.initialize_settings()
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
self.server_port = None
def _init_connection_manager(self, seek_head_blob_first=False):
# this import is requierd here so utils.call_later is replaced by self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(self.downloader, self.rate_limiter,
[self.primary_request_creator], [])
self.connection_manager.seek_head_blob_first = seek_head_blob_first
self.connection_manager._start()
def tearDown(self):
if self.server_port is not None:
self.server_port.stopListening()
self.connection_manager.stop()
conf.settings = None
@defer.inlineCallbacks
def test_success(self):
self._init_connection_manager()
# test to see that if we setup a server, we get a connection
self.server = MocServerProtocolFactory(self.clock)
self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(1, self.TEST_PEER.success_count)
self.assertEqual(0, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_server_with_improper_reply(self):
self._init_connection_manager()
self.server = MocServerProtocolFactory(self.clock, is_good=False)
self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_non_existing_server(self):
# Test to see that if we don't setup a server, we don't get a connection
self._init_connection_manager()
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertFalse(connection_made)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_parallel_connections(self):
# Test to see that we make two new connections at a manage call,
# without it waiting for the connection to complete
self._init_connection_manager()
test_peer2 = Peer(LOCAL_HOST, PEER_PORT+1)
self.primary_request_creator.peers_to_return = [self.TEST_PEER, test_peer2]
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(2, self.connection_manager.num_peer_connections())
self.assertIn(self.TEST_PEER, self.connection_manager._peer_connections)
self.assertIn(test_peer2, self.connection_manager._peer_connections)
deferred_conn_made_peer1 = self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
deferred_conn_made_peer1.addCallback(lambda conn_made: self.assertFalse(conn_made))
deferred_conn_made_peer2 = self.connection_manager._peer_connections[test_peer2].\
factory.connection_was_made_deferred
deferred_conn_made_peer2.addCallback(lambda conn_made: self.assertFalse(conn_made))
yield deferred_conn_made_peer1
yield deferred_conn_made_peer2
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
self.assertEqual(0, test_peer2.success_count)
self.assertEqual(1, test_peer2.down_count)
@defer.inlineCallbacks
def test_stop(self):
# test to see that when we call stop, the ConnectionManager waits for the
# current manage call to finish, closes connections,
# and removes scheduled manage calls
self._init_connection_manager()
self.connection_manager.manage(schedule_next_call=True)
yield self.connection_manager.stop()
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(None, self.connection_manager._next_manage_call)
@defer.inlineCallbacks
def test_closed_connection_when_server_is_slow(self):
self._init_connection_manager()
self.server = MocServerProtocolFactory(
self.clock, has_moc_query_handler=True, is_delayed=True)
self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(True, connection_made)
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
# test header first seeks
@defer.inlineCallbacks
def test_no_peer_for_head_blob(self):
# test that if we can't find blobs for the head blob,
# it looks at the next unavailable and makes connection
self._init_connection_manager(seek_head_blob_first=True)
self.server = MocServerProtocolFactory(self.clock)
self.server_port = reactor.listenTCP(PEER_PORT, self.server, interface=LOCAL_HOST)
self.primary_request_creator.peers_to_return_head_blob = []
self.primary_request_creator.peers_to_return = [self.TEST_PEER]
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[self.TEST_PEER].\
factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(1, self.TEST_PEER.success_count)
self.assertEqual(0, self.TEST_PEER.down_count)
class TestIntegrationConnectionManager(unittest.TestCase):
def setUp(self):
conf.initialize_settings()
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(
self.downloader, self.rate_limiter, [self.primary_request_creator],
[])
self.connection_manager._start()
self.server_port = None
def tearDown(self):
if self.server_port is not None:
self.server_port.stopListening()
self.connection_manager.stop()
conf.settings = None
@defer.inlineCallbacks
def test_success(self):
# test to see that if we setup a server, we get a connection
self.server = MocServerProtocolFactory(self.clock)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[
self.TEST_PEER].factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(1, self.TEST_PEER.success_count)
self.assertEqual(0, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_server_with_improper_reply(self):
self.server = MocServerProtocolFactory(self.clock, is_good=False)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[
self.TEST_PEER].factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertTrue(connection_made)
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
@defer.inlineCallbacks
def test_non_existing_server(self):
# Test to see that if we don't setup a server, we don't get a connection
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[
self.TEST_PEER].factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertFalse(connection_made)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
@unittest.SkipTest
@defer.inlineCallbacks
def test_parallel_connections(self):
# Test to see that we make two new connections at a manage call,
# without it waiting for the connection to complete
test_peer2 = Peer(LOCAL_HOST, PEER_PORT + 1)
self.primary_request_creator.peers_to_return = [
self.TEST_PEER, test_peer2
]
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(2, self.connection_manager.num_peer_connections())
self.assertIn(self.TEST_PEER,
self.connection_manager._peer_connections)
self.assertIn(test_peer2, self.connection_manager._peer_connections)
connection_made = yield self.connection_manager._peer_connections[
self.TEST_PEER].factory.connection_was_made_deferred
self.assertFalse(connection_made)
self.assertEqual(1, self.connection_manager.num_peer_connections())
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
connection_made = yield self.connection_manager._peer_connections[
test_peer2].factory.connection_was_made_deferred
self.assertFalse(connection_made)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(0, test_peer2.success_count)
self.assertEqual(1, test_peer2.down_count)
@defer.inlineCallbacks
def test_stop(self):
# test to see that when we call stop, the ConnectionManager waits for the
# current manage call to finish, closes connections,
# and removes scheduled manage calls
self.connection_manager.manage(schedule_next_call=True)
yield self.connection_manager.stop()
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(None, self.connection_manager._next_manage_call)
@defer.inlineCallbacks
def test_closed_connection_when_server_is_slow(self):
self.server = MocServerProtocolFactory(self.clock,
has_moc_query_handler=True,
is_delayed=True)
self.server_port = reactor.listenTCP(PEER_PORT,
self.server,
interface=LOCAL_HOST)
yield self.connection_manager.manage(schedule_next_call=False)
self.assertEqual(1, self.connection_manager.num_peer_connections())
connection_made = yield self.connection_manager._peer_connections[
self.TEST_PEER].factory.connection_was_made_deferred
self.assertEqual(0, self.connection_manager.num_peer_connections())
self.assertEqual(True, connection_made)
self.assertEqual(0, self.TEST_PEER.success_count)
self.assertEqual(1, self.TEST_PEER.down_count)