def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
self.blob_manager = TempBlobManager(self.hash_announcer)
else:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir,
self.db_dir)
if self.blob_tracker is None:
self.blob_tracker = self.blob_tracker_class(self.blob_manager,
self.peer_finder,
self.dht_node)
if self.payment_rate_manager is None:
self.payment_rate_manager = self.payment_rate_manager_class(
self.base_payment_rate_manager,
self.blob_tracker,
self.is_generous)
self.rate_limiter.start()
d1 = self.blob_manager.setup()
d2 = self.wallet.start()
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
dl.addCallback(lambda _: self.blob_tracker.start())
return dl
def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
raise Exception(
"TempBlobManager is no longer supported, specify BlobManager or db_dir"
)
else:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir,
self.storage)
if self.blob_tracker is None:
self.blob_tracker = self.blob_tracker_class(
self.blob_manager, self.peer_finder, self.dht_node)
if self.payment_rate_manager is None:
self.payment_rate_manager = self.payment_rate_manager_class(
self.base_payment_rate_manager, self.blob_tracker,
self.is_generous)
self.rate_limiter.start()
d = self.storage.setup()
d.addCallback(lambda _: self.wallet.start())
d.addCallback(lambda _: self.blob_tracker.start())
return d
def start(self):
use_epoll_on_linux()
from twisted.internet import reactor
self.reactor = reactor
logging.debug("Starting the uploader")
Random.atfork()
r = random.Random()
r.seed("start_lbry_uploader")
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = RateLimiter()
self.sd_identifier = StreamDescriptorIdentifier()
db_dir = "server"
os.mkdir(db_dir)
self.session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=db_dir, lbryid="abcd",
peer_finder=peer_finder, hash_announcer=hash_announcer, peer_port=5553,
use_upnp=False, rate_limiter=rate_limiter, wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
dht_node_class=Node, is_generous=self.is_generous)
stream_info_manager = TempEncryptedFileMetadataManager()
self.lbry_file_manager = EncryptedFileManager(
self.session, stream_info_manager, self.sd_identifier)
if self.ul_rate_limit is not None:
self.session.rate_limiter.set_ul_limit(self.ul_rate_limit)
reactor.callLater(1, self.start_all)
if not reactor.running:
reactor.run()
def __init__(self, clock, is_good=True, is_delayed=False, has_moc_query_handler=True):
self.rate_limiter = RateLimiter()
query_handler_factory = MocQueryHandlerFactory(clock, is_good, is_delayed)
if has_moc_query_handler:
self.query_handler_factories = {
query_handler_factory.get_primary_query_identifier():query_handler_factory
}
else:
self.query_handler_factories = {}
self.peer_manager = PeerManager()
def setUp(self):
conf.initialize_settings(False)
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
self.server_port = None
def setUp(self):
conf.initialize_settings()
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
utils.call_later = self.clock.callLater
from lbrynet.core.client.ConnectionManager import ConnectionManager
self.connection_manager = ConnectionManager(
self.downloader, self.rate_limiter, [self.primary_request_creator],
[])
self.connection_manager._start()
self.server_port = None
def setup(self):
init_conf_windows()
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.rate_limiter = RateLimiter()
if self.ul_rate_limit is not None:
self.rate_limiter.set_ul_limit(self.ul_rate_limit)
self.prm = OnlyFreePaymentsManager()
self.storage = SQLiteStorage(self.db_dir)
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage)
self.lbry_file_manager = EncryptedFileManager(
FakePeerFinder(5553, self.peer_manager, 1), self.rate_limiter,
self.blob_manager, self.wallet, self.prm, self.storage,
StreamDescriptorIdentifier())
yield self.storage.setup()
yield self.blob_manager.setup()
yield self.lbry_file_manager.setup()
query_handler_factories = {
1:
BlobAvailabilityHandlerFactory(self.blob_manager),
2:
BlobRequestHandlerFactory(self.blob_manager, self.wallet, self.prm,
None),
3:
self.wallet.get_wallet_info_query_handler_factory(),
}
server_factory = ServerProtocolFactory(self.rate_limiter,
query_handler_factories,
self.peer_manager)
self.server_port = reactor.listenTCP(5553,
server_factory,
interface="localhost")
test_file = GenFile(self.file_size,
b''.join([chr(i) for i in xrange(0, 64, 6)]))
lbry_file = yield create_lbry_file(self.blob_manager, self.storage,
self.prm, self.lbry_file_manager,
"test_file", test_file)
defer.returnValue(lbry_file.sd_hash)
def setUp(self):
mocks.mock_conf_settings(self)
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 1)
self.rate_limiter = RateLimiter()
self.prm = OnlyFreePaymentsManager()
self.storage = SQLiteStorage(self.db_dir)
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage)
self.sd_identifier = StreamDescriptorIdentifier()
self.lbry_file_manager = EncryptedFileManager(
self.peer_finder, self.rate_limiter, self.blob_manager,
self.wallet, self.prm, self.storage, self.sd_identifier)
self.uploader = LbryUploader(5209343)
self.sd_hash = yield self.uploader.setup()
yield self.storage.setup()
yield self.blob_manager.setup()
yield self.lbry_file_manager.setup()
yield add_lbry_file_to_sd_identifier(self.sd_identifier)
def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
self.blob_manager = TempBlobManager(self.hash_announcer)
else:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir, self.db_dir)
self.rate_limiter.start()
d1 = self.blob_manager.setup()
d2 = self.wallet.start()
dl = defer.DeferredList([d1, d2],
fireOnOneErrback=True,
consumeErrors=True)
dl.addErrback(lambda err: err.value.subFailure)
return dl
def start(self):
use_epoll_on_linux()
init_conf_windows()
from twisted.internet import reactor
self.reactor = reactor
logging.debug("Starting the uploader")
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = RateLimiter()
self.sd_identifier = StreamDescriptorIdentifier()
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.session = Session(conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=self.db_dir,
blob_dir=self.blob_dir,
node_id="abcd",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
peer_port=5553,
dht_node_port=4445,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
dht_node_class=FakeNode,
is_generous=self.is_generous,
external_ip="127.0.0.1")
self.lbry_file_manager = EncryptedFileManager(self.session,
self.sd_identifier)
if self.ul_rate_limit is not None:
self.session.rate_limiter.set_ul_limit(self.ul_rate_limit)
reactor.callLater(1, self.start_all)
if not reactor.running:
reactor.run()
def start_blob_uploader(blob_hash_queue,
kill_event,
dead_event,
slow,
is_generous=False):
use_epoll_on_linux()
from twisted.internet import reactor
logging.debug("Starting the uploader")
Random.atfork()
wallet = FakeWallet()
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = RateLimiter()
if slow is True:
peer_port = 5553
db_dir = "server1"
else:
peer_port = 5554
db_dir = "server2"
blob_dir = os.path.join(db_dir, "blobfiles")
os.mkdir(db_dir)
os.mkdir(blob_dir)
session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
lbryid="efgh",
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=blob_dir,
peer_port=peer_port,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1])
if slow is True:
session.rate_limiter.set_ul_limit(2**11)
def start_all():
d = session.setup()
d.addCallback(lambda _: start_server())
d.addCallback(lambda _: create_single_blob())
d.addCallback(put_blob_hash_on_queue)
def print_error(err):
logging.critical("Server error: %s", err.getErrorMessage())
d.addErrback(print_error)
return d
def start_server():
server_port = None
query_handler_factories = {
1:
BlobAvailabilityHandlerFactory(session.blob_manager),
2:
BlobRequestHandlerFactory(session.blob_manager, session.wallet,
session.payment_rate_manager, None),
3:
session.wallet.get_wallet_info_query_handler_factory(),
}
server_factory = ServerProtocolFactory(session.rate_limiter,
query_handler_factories,
session.peer_manager)
server_port = reactor.listenTCP(peer_port, server_factory)
logging.debug("Started listening")
def kill_server():
ds = []
ds.append(session.shut_down())
if server_port:
ds.append(server_port.stopListening())
kill_check.stop()
dead_event.set()
dl = defer.DeferredList(ds)
dl.addCallback(lambda _: reactor.stop())
return dl
def check_for_kill():
if kill_event.is_set():
kill_server()
kill_check = task.LoopingCall(check_for_kill)
kill_check.start(1.0)
return True
def create_single_blob():
blob_creator = session.blob_manager.get_blob_creator()
blob_creator.write("0" * 2**21)
return blob_creator.close()
def put_blob_hash_on_queue(blob_hash):
logging.debug("Telling the client to start running. Blob hash: %s",
str(blob_hash))
blob_hash_queue.put(blob_hash)
logging.debug("blob hash has been added to the queue")
reactor.callLater(1, start_all)
if not reactor.running:
reactor.run()
def start_lbry_reuploader(sd_hash,
kill_event,
dead_event,
ready_event,
n,
ul_rate_limit=None,
is_generous=False):
use_epoll_on_linux()
from twisted.internet import reactor
logging.debug("Starting the uploader")
Random.atfork()
r = random.Random()
r.seed("start_lbry_reuploader")
wallet = FakeWallet()
peer_port = 5553 + n
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = RateLimiter()
sd_identifier = StreamDescriptorIdentifier()
db_dir = "server_" + str(n)
blob_dir = os.path.join(db_dir, "blobfiles")
os.mkdir(db_dir)
os.mkdir(blob_dir)
session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
lbryid="abcd" + str(n),
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=None,
peer_port=peer_port,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1])
stream_info_manager = TempEncryptedFileMetadataManager()
lbry_file_manager = EncryptedFileManager(session, stream_info_manager,
sd_identifier)
if ul_rate_limit is not None:
session.rate_limiter.set_ul_limit(ul_rate_limit)
def make_downloader(metadata, prm):
info_validator = metadata.validator
options = metadata.options
factories = metadata.factories
chosen_options = [
o.default_value
for o in options.get_downloader_options(info_validator, prm)
]
return factories[0].make_downloader(metadata, chosen_options, prm)
def download_file():
prm = session.payment_rate_manager
d = download_sd_blob(session, sd_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(make_downloader, prm)
d.addCallback(lambda downloader: downloader.start())
return d
def start_transfer():
logging.debug("Starting the transfer")
d = session.setup()
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier))
d.addCallback(lambda _: lbry_file_manager.setup())
d.addCallback(lambda _: download_file())
return d
def start_server():
server_port = None
query_handler_factories = {
1:
BlobAvailabilityHandlerFactory(session.blob_manager),
2:
BlobRequestHandlerFactory(session.blob_manager, session.wallet,
session.payment_rate_manager, None),
3:
session.wallet.get_wallet_info_query_handler_factory(),
}
server_factory = ServerProtocolFactory(session.rate_limiter,
query_handler_factories,
session.peer_manager)
server_port = reactor.listenTCP(peer_port, server_factory)
logging.debug("Started listening")
def kill_server():
ds = []
ds.append(session.shut_down())
ds.append(lbry_file_manager.stop())
if server_port:
ds.append(server_port.stopListening())
kill_check.stop()
dead_event.set()
dl = defer.DeferredList(ds)
dl.addCallback(lambda _: reactor.stop())
return dl
def check_for_kill():
if kill_event.is_set():
kill_server()
kill_check = task.LoopingCall(check_for_kill)
kill_check.start(1.0)
ready_event.set()
logging.debug("set the ready event")
d = task.deferLater(reactor, 1.0, start_transfer)
d.addCallback(lambda _: start_server())
if not reactor.running:
reactor.run()
def __init__(self, component_manager):
Component.__init__(self, component_manager)
self.rate_limiter = RateLimiter()
def start_lbry_reuploader(sd_hash,
kill_event,
dead_event,
ready_event,
n,
ul_rate_limit=None,
is_generous=False):
use_epoll_on_linux()
init_conf_windows()
from twisted.internet import reactor
logging.debug("Starting the uploader")
wallet = FakeWallet()
peer_port = 5553 + n
peer_manager = PeerManager()
peer_finder = FakePeerFinder(5553, peer_manager, 1)
hash_announcer = FakeAnnouncer()
rate_limiter = RateLimiter()
sd_identifier = StreamDescriptorIdentifier()
db_dir, blob_dir = mk_db_and_blob_dir()
session = Session(
conf.ADJUSTABLE_SETTINGS['data_rate'][1],
db_dir=db_dir,
node_id="abcd" + str(n),
dht_node_port=4446,
dht_node_class=FakeNode,
peer_finder=peer_finder,
hash_announcer=hash_announcer,
blob_dir=blob_dir,
peer_port=peer_port,
use_upnp=False,
rate_limiter=rate_limiter,
wallet=wallet,
blob_tracker_class=DummyBlobAvailabilityTracker,
is_generous=conf.ADJUSTABLE_SETTINGS['is_generous_host'][1],
external_ip="127.0.0.1")
lbry_file_manager = EncryptedFileManager(session, sd_identifier)
if ul_rate_limit is not None:
session.rate_limiter.set_ul_limit(ul_rate_limit)
def make_downloader(metadata, prm, download_directory):
factories = metadata.factories
return factories[0].make_downloader(metadata,
prm.min_blob_data_payment_rate,
prm, download_directory)
def download_file():
prm = session.payment_rate_manager
d = download_sd_blob(session, sd_hash, prm)
d.addCallback(sd_identifier.get_metadata_for_sd_blob)
d.addCallback(make_downloader, prm, db_dir)
d.addCallback(lambda downloader: downloader.start())
return d
def start_transfer():
logging.debug("Starting the transfer")
d = session.setup()
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier))
d.addCallback(lambda _: lbry_file_manager.setup())
d.addCallback(lambda _: download_file())
return d
def start_server():
server_port = None
query_handler_factories = {
1:
BlobAvailabilityHandlerFactory(session.blob_manager),
2:
BlobRequestHandlerFactory(session.blob_manager, session.wallet,
session.payment_rate_manager, None),
3:
session.wallet.get_wallet_info_query_handler_factory(),
}
server_factory = ServerProtocolFactory(session.rate_limiter,
query_handler_factories,
session.peer_manager)
server_port = reactor.listenTCP(peer_port, server_factory)
logging.debug("Started listening")
def kill_server():
ds = []
ds.append(session.shut_down())
ds.append(lbry_file_manager.stop())
if server_port:
ds.append(server_port.stopListening())
ds.append(rm_db_and_blob_dir(db_dir, blob_dir))
kill_check.stop()
dead_event.set()
dl = defer.DeferredList(ds)
dl.addCallback(lambda _: reactor.stop())
return dl
def check_for_kill():
if kill_event.is_set():
kill_server()
kill_check = task.LoopingCall(check_for_kill)
kill_check.start(1.0)
ready_event.set()
logging.debug("set the ready event")
d = task.deferLater(reactor, 1.0, start_transfer)
d.addCallback(lambda _: start_server())
if not reactor.running:
reactor.run()
