def test_create_and_combine_stream(self): wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 2) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() db_dir = "client" blob_dir = os.path.join(db_dir, "blobfiles") os.mkdir(db_dir) os.mkdir(blob_dir) self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, node_id="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, external_ip="127.0.0.1" ) self.stream_info_manager = DBEncryptedFileMetadataManager(self.session.db_dir) self.lbry_file_manager = EncryptedFileManager( self.session, self.stream_info_manager, sd_identifier) def start_lbry_file(lbry_file): logging.debug("Calling lbry_file.start()") d = lbry_file.start() return d def combine_stream(stream_hash): prm = self.session.payment_rate_manager d = self.lbry_file_manager.add_lbry_file(stream_hash, prm) d.addCallback(start_lbry_file) def check_md5_sum(): f = open('test_file') hashsum = MD5.new() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "68959747edc73df45e45db6379dd7b3b") d.addCallback(lambda _: check_md5_sum()) return d @defer.inlineCallbacks def create_stream(): test_file = GenFile(53209343, b''.join([chr(i + 5) for i in xrange(0, 64, 6)])) stream_hash = yield create_lbry_file(self.session, self.lbry_file_manager, "test_file", test_file, suggested_file_name="test_file") yield publish_sd_blob(self.stream_info_manager, self.session.blob_manager, stream_hash) defer.returnValue(stream_hash) d = self.session.setup() d.addCallback(lambda _: self.stream_info_manager.setup()) d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier)) d.addCallback(lambda _: self.lbry_file_manager.setup()) d.addCallback(lambda _: create_stream()) d.addCallback(combine_stream) return d
def test_create_and_combine_stream(self): wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 2) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=self.db_dir, node_id="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=self.blob_dir, peer_port=5553, dht_node_class=mocks.Node, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, external_ip="127.0.0.1" ) self.lbry_file_manager = EncryptedFileManager(self.session, sd_identifier) @defer.inlineCallbacks def create_stream(): test_file = GenFile(53209343, b''.join([chr(i + 5) for i in xrange(0, 64, 6)])) lbry_file = yield create_lbry_file(self.session, self.lbry_file_manager, "test_file", test_file) sd_hash = yield self.session.storage.get_sd_blob_hash_for_stream(lbry_file.stream_hash) self.assertTrue(lbry_file.sd_hash, sd_hash) yield lbry_file.start() f = open('test_file') hashsum = MD5.new() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "68959747edc73df45e45db6379dd7b3b") d = self.session.setup() d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier)) d.addCallback(lambda _: self.lbry_file_manager.setup()) d.addCallback(lambda _: create_stream()) return d
def test_create_stream(self): wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 2) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() self.session = Session(conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=self.db_dir, node_id="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=self.blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=self.is_generous, external_ip="127.0.0.1", dht_node_class=mocks.Node) self.lbry_file_manager = EncryptedFileManager(self.session, sd_identifier) d = self.session.setup() d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier)) d.addCallback(lambda _: self.lbry_file_manager.setup()) def verify_equal(sd_info): self.assertEqual(sd_info, test_create_stream_sd_file) def verify_stream_descriptor_file(stream_hash): d = get_sd_info(self.session.storage, stream_hash, True) d.addCallback(verify_equal) return d def iv_generator(): iv = 0 while 1: iv += 1 yield "%016d" % iv def create_stream(): test_file = GenFile( 5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)])) d = create_lbry_file(self.session, self.lbry_file_manager, "test_file", test_file, key="0123456701234567", iv_generator=iv_generator()) d.addCallback(lambda lbry_file: lbry_file.stream_hash) return d d.addCallback(lambda _: create_stream()) d.addCallback(verify_stream_descriptor_file) return d
def setUp(self): mocks.mock_conf_settings(self) self.tmp_db_dir, self.tmp_blob_dir = mk_db_and_blob_dir() self.wallet = FakeWallet() self.peer_manager = PeerManager() self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2) self.rate_limiter = DummyRateLimiter() self.sd_identifier = StreamDescriptorIdentifier() self.storage = SQLiteStorage(self.tmp_db_dir) self.blob_manager = DiskBlobManager(self.tmp_blob_dir, self.storage) self.prm = OnlyFreePaymentsManager() self.lbry_file_manager = EncryptedFileManager( self.peer_finder, self.rate_limiter, self.blob_manager, self.wallet, self.prm, self.storage, self.sd_identifier) d = self.storage.setup() d.addCallback(lambda _: self.lbry_file_manager.setup()) return d
def setUp(self): mocks.mock_conf_settings(self) self.session = None self.lbry_file_manager = None self.is_generous = True self.db_dir = tempfile.mkdtemp() self.blob_dir = os.path.join(self.db_dir, "blobfiles") os.mkdir(self.blob_dir) self.dht_node = FakeNode() self.wallet = FakeWallet() self.peer_manager = PeerManager() self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2) self.rate_limiter = DummyRateLimiter() self.sd_identifier = StreamDescriptorIdentifier() self.storage = SQLiteStorage(self.db_dir) self.blob_manager = DiskBlobManager(self.blob_dir, self.storage, self.dht_node._dataStore) self.prm = OnlyFreePaymentsManager() self.lbry_file_manager = EncryptedFileManager( self.peer_finder, self.rate_limiter, self.blob_manager, self.wallet, self.prm, self.storage, self.sd_identifier) d = self.storage.setup() d.addCallback(lambda _: self.lbry_file_manager.setup()) return d
def test_multiple_uploaders(self): sd_hash_queue = Queue() num_uploaders = 3 kill_event = Event() dead_events = [Event() for _ in range(num_uploaders)] ready_events = [Event() for _ in range(1, num_uploaders)] lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_events[0], 5209343, 9373419, 2**22) uploader = Process(target=lbry_uploader.start) uploader.start() self.server_processes.append(uploader) logging.debug("Testing multiple uploaders") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, num_uploaders) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() db_dir = "client" blob_dir = os.path.join(db_dir, "blobfiles") os.mkdir(db_dir) os.mkdir(blob_dir) self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=None, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1]) self.stream_info_manager = TempEncryptedFileMetadataManager() self.lbry_file_manager = EncryptedFileManager(self.session, self.stream_info_manager, sd_identifier) def start_additional_uploaders(sd_hash): for i in range(1, num_uploaders): uploader = Process(target=start_lbry_reuploader, args=(sd_hash, kill_event, dead_events[i], ready_events[i - 1], i, 2**10)) uploader.start() self.server_processes.append(uploader) return defer.succeed(True) def wait_for_ready_events(): return defer.DeferredList([ self.wait_for_event(ready_event, 60) for ready_event in ready_events ]) def make_downloader(metadata, prm): info_validator = metadata.validator options = metadata.options factories = metadata.factories chosen_options = [ o.default_value for o in options.get_downloader_options(info_validator, prm) ] return factories[0].make_downloader(metadata, chosen_options, prm) def download_file(sd_hash): prm = self.session.payment_rate_manager d = download_sd_blob(self.session, sd_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(make_downloader, prm) d.addCallback(lambda downloader: downloader.start()) return d def check_md5_sum(): f = open('test_file') hashsum = MD5.new() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "e5941d615f53312fd66638239c1f90d5") def start_transfer(sd_hash): logging.debug("Starting the transfer") d = start_additional_uploaders(sd_hash) d.addCallback(lambda _: wait_for_ready_events()) d.addCallback(lambda _: self.session.setup()) d.addCallback( lambda _: add_lbry_file_to_sd_identifier(sd_identifier)) d.addCallback(lambda _: self.lbry_file_manager.setup()) d.addCallback(lambda _: download_file(sd_hash)) d.addCallback(lambda _: check_md5_sum()) return d def stop(arg): if isinstance(arg, Failure): logging.debug("Client is stopping due to an error. Error: %s", arg.getTraceback()) else: logging.debug("Client is stopping normally.") kill_event.set() logging.debug("Set the kill event") d = defer.DeferredList([ self.wait_for_event(dead_event, 15) for dead_event in dead_events ]) def print_shutting_down(): logging.info("Client is shutting down") d.addCallback(lambda _: print_shutting_down()) d.addCallback(lambda _: arg) return d d = self.wait_for_hash_from_queue(sd_hash_queue) d.addCallback(start_transfer) d.addBoth(stop) return d
def test_double_download(self): sd_hash_queue = Queue() kill_event = Event() dead_event = Event() lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_event, 5209343) uploader = Process(target=lbry_uploader.start) uploader.start() self.server_processes.append(uploader) logging.debug("Testing double download") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 1) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() downloaders = [] db_dir = "client" blob_dir = os.path.join(db_dir, "blobfiles") os.mkdir(db_dir) os.mkdir(blob_dir) self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1]) self.stream_info_manager = DBEncryptedFileMetadataManager( self.session.db_dir) self.lbry_file_manager = EncryptedFileManager(self.session, self.stream_info_manager, sd_identifier) @defer.inlineCallbacks def make_downloader(metadata, prm): info_validator = metadata.validator options = metadata.options factories = metadata.factories chosen_options = [ o.default_value for o in options.get_downloader_options(info_validator, prm) ] downloader = yield factories[0].make_downloader( metadata, chosen_options, prm) defer.returnValue(downloader) def append_downloader(downloader): downloaders.append(downloader) return downloader @defer.inlineCallbacks def download_file(sd_hash): prm = self.session.payment_rate_manager sd_blob = yield download_sd_blob(self.session, sd_hash, prm) metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob) downloader = yield make_downloader(metadata, prm) downloaders.append(downloader) finished_value = yield downloader.start() defer.returnValue(finished_value) def check_md5_sum(): f = open('test_file') hashsum = MD5.new() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "4ca2aafb4101c1e42235aad24fbb83be") def delete_lbry_file(): logging.debug("deleting the file") d = self.lbry_file_manager.delete_lbry_file(downloaders[0]) d.addCallback( lambda _: self.lbry_file_manager.get_count_for_stream_hash( downloaders[0].stream_hash)) d.addCallback(lambda c: self.stream_info_manager.delete_stream( downloaders[1].stream_hash) if c == 0 else True) return d def check_lbry_file(): d = downloaders[1].status() d.addCallback(lambda _: downloaders[1].status()) def check_status_report(status_report): self.assertEqual(status_report.num_known, status_report.num_completed) self.assertEqual(status_report.num_known, 3) d.addCallback(check_status_report) return d @defer.inlineCallbacks def start_transfer(sd_hash): logging.debug("Starting the transfer") yield self.session.setup() yield self.stream_info_manager.setup() yield add_lbry_file_to_sd_identifier(sd_identifier) yield self.lbry_file_manager.setup() yield download_file(sd_hash) yield check_md5_sum() yield download_file(sd_hash) yield check_lbry_file() yield delete_lbry_file() def stop(arg): if isinstance(arg, Failure): logging.debug("Client is stopping due to an error. Error: %s", arg.getTraceback()) else: logging.debug("Client is stopping normally.") kill_event.set() logging.debug("Set the kill event") d = self.wait_for_event(dead_event, 15) def print_shutting_down(): logging.info("Client is shutting down") d.addCallback(lambda _: print_shutting_down()) d.addCallback(lambda _: arg) return d d = self.wait_for_hash_from_queue(sd_hash_queue) d.addCallback(start_transfer) d.addBoth(stop) return d
def test_last_blob_retrieval(self): kill_event = Event() dead_event_1 = Event() blob_hash_queue_1 = Queue() blob_hash_queue_2 = Queue() fast_uploader = Process(target=start_blob_uploader, args=(blob_hash_queue_1, kill_event, dead_event_1, False)) fast_uploader.start() self.server_processes.append(fast_uploader) dead_event_2 = Event() slow_uploader = Process(target=start_blob_uploader, args=(blob_hash_queue_2, kill_event, dead_event_2, True)) slow_uploader.start() self.server_processes.append(slow_uploader) logging.debug("Testing transfer") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 2) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() db_dir = "client" blob_dir = os.path.join(db_dir, "blobfiles") os.mkdir(db_dir) os.mkdir(blob_dir) self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1]) d1 = self.wait_for_hash_from_queue(blob_hash_queue_1) d2 = self.wait_for_hash_from_queue(blob_hash_queue_2) d = defer.DeferredList([d1, d2], fireOnOneErrback=True) def get_blob_hash(results): self.assertEqual(results[0][1], results[1][1]) return results[0][1] d.addCallback(get_blob_hash) def download_blob(blob_hash): prm = self.session.payment_rate_manager downloader = StandaloneBlobDownloader(blob_hash, self.session.blob_manager, peer_finder, rate_limiter, prm, wallet) d = downloader.download() return d def start_transfer(blob_hash): logging.debug("Starting the transfer") d = self.session.setup() d.addCallback(lambda _: download_blob(blob_hash)) return d d.addCallback(start_transfer) def stop(arg): if isinstance(arg, Failure): logging.debug("Client is stopping due to an error. Error: %s", arg.getTraceback()) else: logging.debug("Client is stopping normally.") kill_event.set() logging.debug("Set the kill event") d1 = self.wait_for_event(dead_event_1, 15) d2 = self.wait_for_event(dead_event_2, 15) dl = defer.DeferredList([d1, d2]) def print_shutting_down(): logging.info("Client is shutting down") dl.addCallback(lambda _: print_shutting_down()) dl.addCallback(lambda _: arg) return dl d.addBoth(stop) return d
def __init__(self): self._payment_rate_manager = OnlyFreePaymentsManager() self._rate_limiter = DummyRateLimiter() self._wallet = None self._blob_manager = None
def test_lbry_transfer(self): sd_hash_queue = Queue() kill_event = Event() dead_event = Event() lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_event, 5209343) uploader = Process(target=lbry_uploader.start) uploader.start() self.server_processes.append(uploader) logging.debug("Testing transfer") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 1) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() db_dir, blob_dir = mk_db_and_blob_dir() self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, dht_node_class=Node, is_generous=self.is_generous) self.stream_info_manager = TempEncryptedFileMetadataManager() self.lbry_file_manager = EncryptedFileManager( self.session, self.stream_info_manager, sd_identifier) def make_downloader(metadata, prm): info_validator = metadata.validator options = metadata.options factories = metadata.factories chosen_options = [ o.default_value for o in options.get_downloader_options(info_validator, prm) ] return factories[0].make_downloader(metadata, chosen_options, prm) def download_file(sd_hash): prm = self.session.payment_rate_manager d = download_sd_blob(self.session, sd_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(make_downloader, prm) d.addCallback(lambda downloader: downloader.start()) return d def check_md5_sum(): f = open('test_file') hashsum = MD5.new() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "4ca2aafb4101c1e42235aad24fbb83be") @defer.inlineCallbacks def start_transfer(sd_hash): logging.debug("Starting the transfer") yield self.session.setup() yield add_lbry_file_to_sd_identifier(sd_identifier) yield self.lbry_file_manager.setup() yield download_file(sd_hash) yield check_md5_sum() def stop(arg): if isinstance(arg, Failure): logging.debug("Client is stopping due to an error. Error: %s", arg.getTraceback()) else: logging.debug("Client is stopping normally.") kill_event.set() logging.debug("Set the kill event") d = self.wait_for_event(dead_event, 15) def print_shutting_down(): logging.info("Client is shutting down") d.addCallback(lambda _: print_shutting_down()) d.addCallback(lambda _: rm_db_and_blob_dir(db_dir, blob_dir)) d.addCallback(lambda _: arg) return d d = self.wait_for_hash_from_queue(sd_hash_queue) d.addCallback(start_transfer) d.addBoth(stop) return d
def setUp(self): self.reflector_port = None self.port = None mocks.mock_conf_settings(self) self.server_db_dir, self.server_blob_dir = mk_db_and_blob_dir() self.client_db_dir, self.client_blob_dir = mk_db_and_blob_dir() prm = OnlyFreePaymentsManager() wallet = mocks.Wallet() peer_manager = PeerManager.PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) self.server_storage = SQLiteStorage(self.server_db_dir) self.server_blob_manager = BlobManager.DiskBlobManager( self.server_blob_dir, self.server_storage) self.client_storage = SQLiteStorage(self.client_db_dir) self.client_blob_manager = BlobManager.DiskBlobManager( self.client_blob_dir, self.client_storage) self.server_lbry_file_manager = EncryptedFileManager( peer_finder, DummyRateLimiter(), self.server_blob_manager, wallet, prm, self.server_storage, StreamDescriptor.StreamDescriptorIdentifier()) self.client_lbry_file_manager = EncryptedFileManager( peer_finder, DummyRateLimiter(), self.client_blob_manager, wallet, prm, self.client_storage, StreamDescriptor.StreamDescriptorIdentifier()) self.expected_blobs = [ ('dc4708f76a5e7af0f1cae0ee96b824e2ed9250c9346c093b' '441f0a20d3607c17948b6fcfb4bc62020fe5286693d08586', 2097152), ('f4067522c1b49432a2a679512e3917144317caa1abba0c04' '1e0cd2cf9f635d4cf127ce1824fa04189b63916174951f70', 2097152), ('305486c434260484fcb2968ce0e963b72f81ba56c11b08b1' 'af0789b55b44d78422600f9a38e3cf4f2e9569897e5646a9', 1015056), ] d = self.server_storage.setup() d.addCallback(lambda _: self.server_blob_manager.setup()) d.addCallback(lambda _: self.server_lbry_file_manager.setup()) d.addCallback(lambda _: self.client_storage.setup()) d.addCallback(lambda _: self.client_blob_manager.setup()) d.addCallback(lambda _: self.client_lbry_file_manager.setup()) @defer.inlineCallbacks def verify_equal(sd_info, stream_hash): self.assertDictEqual(mocks.create_stream_sd_file, sd_info) sd_hash = yield self.client_storage.get_sd_blob_hash_for_stream( stream_hash) defer.returnValue(sd_hash) def save_sd_blob_hash(sd_hash): self.sd_hash = sd_hash self.expected_blobs.append((sd_hash, 923)) def verify_stream_descriptor_file(stream_hash): self.stream_hash = stream_hash d = get_sd_info(self.client_storage, stream_hash, True) d.addCallback(verify_equal, stream_hash) d.addCallback(save_sd_blob_hash) return d def create_stream(): test_file = mocks.GenFile(5209343, bytes((i + 3) for i in range(0, 64, 6))) d = EncryptedFileCreator.create_lbry_file( self.client_blob_manager, self.client_storage, prm, self.client_lbry_file_manager, "test_file", test_file, key=b"0123456701234567", iv_generator=iv_generator()) d.addCallback(lambda lbry_file: lbry_file.stream_hash) return d def start_server(): server_factory = reflector.ServerFactory( peer_manager, self.server_blob_manager, self.server_lbry_file_manager) from twisted.internet import reactor port = 8943 while self.reflector_port is None: try: self.reflector_port = reactor.listenTCP( port, server_factory) self.port = port except error.CannotListenError: port += 1 d.addCallback(lambda _: create_stream()) d.addCallback(verify_stream_descriptor_file) d.addCallback(lambda _: start_server()) return d
def test_live_transfer(self): sd_hash_queue = Queue() kill_event = Event() dead_event = Event() server_args = (sd_hash_queue, kill_event, dead_event) server = Process(target=start_live_server, args=server_args) server.start() self.server_processes.append(server) wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 1) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() db_dir = "client" os.mkdir(db_dir) self.session = Session(conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=None, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, dht_node_class=Node) self.stream_info_manager = TempLiveStreamMetadataManager( hash_announcer) d = self.wait_for_hash_from_queue(sd_hash_queue) def create_downloader(metadata, prm): info_validator = metadata.validator options = metadata.options factories = metadata.factories chosen_options = [ o.default_value for o in options.get_downloader_options(info_validator, prm) ] return factories[0].make_downloader(metadata, chosen_options, prm) def start_lbry_file(lbry_file): lbry_file = lbry_file return lbry_file.start() def download_stream(sd_blob_hash): prm = self.session.payment_rate_manager d = download_sd_blob(self.session, sd_blob_hash, prm) d.addCallback(sd_identifier.get_metadata_for_sd_blob) d.addCallback(create_downloader, prm) d.addCallback(start_lbry_file) return d def do_download(sd_blob_hash): logging.debug("Starting the download") d = self.session.setup() d.addCallback(lambda _: enable_live_stream()) d.addCallback(lambda _: download_stream(sd_blob_hash)) return d def enable_live_stream(): add_live_stream_to_sd_identifier(sd_identifier, self.session.payment_rate_manager) add_full_live_stream_downloader_to_sd_identifier( self.session, self.stream_info_manager, sd_identifier, self.session.payment_rate_manager) d.addCallback(do_download) def check_md5_sum(): f = open('test_file') hashsum = MD5.new() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "215b177db8eed86d028b37e5cbad55c7") d.addCallback(lambda _: check_md5_sum()) def stop(arg): if isinstance(arg, Failure): logging.debug("Client is stopping due to an error. Error: %s", arg.getTraceback()) else: logging.debug("Client is stopping normally.") kill_event.set() logging.debug("Set the kill event") d = self.wait_for_event(dead_event, 15) def print_shutting_down(): logging.info("Client is shutting down") d.addCallback(lambda _: print_shutting_down()) d.addCallback(lambda _: arg) return d d.addBoth(stop) return d
def start_live_server(sd_hash_queue, kill_event, dead_event): use_epoll_on_linux() from twisted.internet import reactor logging.debug("In start_server.") Random.atfork() r = random.Random() r.seed("start_live_server") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 1) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() db_dir = "server" os.mkdir(db_dir) session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1]) stream_info_manager = DBLiveStreamMetadataManager(session.db_dir, hash_announcer) logging.debug("Created the session") server_port = [] def start_listening(): logging.debug("Starting the server protocol") query_handler_factories = { 1: CryptBlobInfoQueryHandlerFactory(stream_info_manager, session.wallet, session.payment_rate_manager), 2: BlobRequestHandlerFactory(session.blob_manager, session.wallet, session.payment_rate_manager, analytics.Track()), 3: session.wallet.get_wallet_info_query_handler_factory() } server_factory = ServerProtocolFactory(session.rate_limiter, query_handler_factories, session.peer_manager) server_port.append(reactor.listenTCP(5553, server_factory)) logging.debug("Server protocol has started") def create_stream(): logging.debug("Making the live stream") test_file = GenFile(5209343, b''.join([chr(i + 2) for i in xrange(0, 64, 6)])) stream_creator_helper = FileLiveStreamCreator(session.blob_manager, stream_info_manager, "test_file", test_file) d = stream_creator_helper.setup() d.addCallback( lambda _: stream_creator_helper.publish_stream_descriptor()) d.addCallback(put_sd_hash_on_queue) d.addCallback(lambda _: stream_creator_helper.start_streaming()) return d def put_sd_hash_on_queue(sd_hash): logging.debug("Telling the client to start running. Stream hash: %s", str(sd_hash)) sd_hash_queue.put(sd_hash) logging.debug("sd hash has been added to the queue") def set_dead_event(): logging.debug("Setting the dead event") dead_event.set() def print_error(err): logging.debug("An error occurred during shutdown: %s", err.getTraceback()) def stop_reactor(): logging.debug("Server is stopping its reactor") reactor.stop() def shut_down(arg): logging.debug("Shutting down") if isinstance(arg, Failure): logging.error("Shut down is due to an error: %s", arg.getTraceback()) d = defer.maybeDeferred(server_port[0].stopListening) d.addErrback(print_error) d.addCallback(lambda _: session.shut_down()) d.addCallback(lambda _: stream_info_manager.stop()) d.addErrback(print_error) d.addCallback(lambda _: set_dead_event()) d.addErrback(print_error) d.addCallback(lambda _: reactor.callLater(0, stop_reactor)) d.addErrback(print_error) return d def wait_for_kill_event(): d = defer.Deferred() def check_for_kill(): if kill_event.is_set(): logging.debug("Kill event has been found set") kill_check.stop() d.callback(True) kill_check = task.LoopingCall(check_for_kill) kill_check.start(1.0) return d def enable_live_stream(): add_live_stream_to_sd_identifier(sd_identifier, session.base_payment_rate_manager) add_full_live_stream_downloader_to_sd_identifier( session, stream_info_manager, sd_identifier, session.base_payment_rate_manager) def run_server(): d = session.setup() d.addCallback(lambda _: stream_info_manager.setup()) d.addCallback(lambda _: enable_live_stream()) d.addCallback(lambda _: start_listening()) d.addCallback(lambda _: create_stream()) d.addCallback(lambda _: wait_for_kill_event()) d.addBoth(shut_down) return d reactor.callLater(1, run_server) if not reactor.running: reactor.run()
def test_double_download(self): sd_hash_queue = Queue() kill_event = Event() dead_event = Event() lbry_uploader = LbryUploader(sd_hash_queue, kill_event, dead_event, 5209343) uploader = Process(target=lbry_uploader.start) uploader.start() self.server_processes.append(uploader) logging.debug("Testing double download") wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 1) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() downloaders = [] db_dir, blob_dir = mk_db_and_blob_dir() self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, node_id="abcd", peer_finder=peer_finder, dht_node_port=4445, dht_node_class=FakeNode, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1], external_ip="127.0.0.1") self.lbry_file_manager = EncryptedFileManager(self.session, sd_identifier) @defer.inlineCallbacks def make_downloader(metadata, prm): factories = metadata.factories downloader = yield factories[0].make_downloader( metadata, prm.min_blob_data_payment_rate, prm, db_dir) defer.returnValue(downloader) @defer.inlineCallbacks def download_file(sd_hash): prm = self.session.payment_rate_manager sd_blob = yield download_sd_blob(self.session, sd_hash, prm) metadata = yield sd_identifier.get_metadata_for_sd_blob(sd_blob) downloader = yield make_downloader(metadata, prm) downloaders.append(downloader) yield downloader.start() defer.returnValue(downloader) def check_md5_sum(): f = open(os.path.join(db_dir, 'test_file')) hashsum = md5() hashsum.update(f.read()) self.assertEqual(hashsum.hexdigest(), "4ca2aafb4101c1e42235aad24fbb83be") def delete_lbry_file(downloader): logging.debug("deleting the file") return self.lbry_file_manager.delete_lbry_file(downloader) def check_lbry_file(downloader): d = downloader.status() def check_status_report(status_report): self.assertEqual(status_report.num_known, status_report.num_completed) self.assertEqual(status_report.num_known, 3) d.addCallback(check_status_report) return d @defer.inlineCallbacks def start_transfer(sd_hash): # download a file, delete it, and download it again logging.debug("Starting the transfer") yield self.session.setup() yield add_lbry_file_to_sd_identifier(sd_identifier) yield self.lbry_file_manager.setup() downloader = yield download_file(sd_hash) yield check_md5_sum() yield check_lbry_file(downloader) yield delete_lbry_file(downloader) downloader = yield download_file(sd_hash) yield check_lbry_file(downloader) yield check_md5_sum() yield delete_lbry_file(downloader) def stop(arg): if isinstance(arg, Failure): logging.debug("Client is stopping due to an error. Error: %s", arg.getTraceback()) else: logging.debug("Client is stopping normally.") kill_event.set() logging.debug("Set the kill event") d = self.wait_for_event(dead_event, 15) def print_shutting_down(): logging.info("Client is shutting down") d.addCallback(lambda _: print_shutting_down()) d.addCallback(lambda _: rm_db_and_blob_dir(db_dir, blob_dir)) d.addCallback(lambda _: arg) return d d = self.wait_for_hash_from_queue(sd_hash_queue) d.addCallback(start_transfer) d.addBoth(stop) return d
def test_create_stream(self): wallet = FakeWallet() peer_manager = PeerManager() peer_finder = FakePeerFinder(5553, peer_manager, 2) hash_announcer = FakeAnnouncer() rate_limiter = DummyRateLimiter() sd_identifier = StreamDescriptorIdentifier() db_dir = "client" blob_dir = os.path.join(db_dir, "blobfiles") os.mkdir(db_dir) os.mkdir(blob_dir) self.session = Session(settings.data_rate, db_dir=db_dir, lbryid="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, blob_dir=blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, is_generous=self.is_generous) self.stream_info_manager = TempEncryptedFileMetadataManager() self.lbry_file_manager = EncryptedFileManager(self.session, self.stream_info_manager, sd_identifier) d = self.session.setup() d.addCallback(lambda _: self.stream_info_manager.setup()) d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier)) d.addCallback(lambda _: self.lbry_file_manager.setup()) def verify_equal(sd_info): self.assertEqual(sd_info, test_create_stream_sd_file) def verify_stream_descriptor_file(stream_hash): d = get_sd_info(self.lbry_file_manager.stream_info_manager, stream_hash, True) d.addCallback(verify_equal) return d def iv_generator(): iv = 0 while 1: iv += 1 yield "%016d" % iv def create_stream(): test_file = GenFile( 5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)])) d = create_lbry_file(self.session, self.lbry_file_manager, "test_file", test_file, key="0123456701234567", iv_generator=iv_generator()) return d d.addCallback(lambda _: create_stream()) d.addCallback(verify_stream_descriptor_file) return d