def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
self.blob_manager = TempBlobManager(self.hash_announcer)
else:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir,
self.db_dir)
if self.blob_tracker is None:
self.blob_tracker = self.blob_tracker_class(self.blob_manager,
self.peer_finder,
self.dht_node)
if self.payment_rate_manager is None:
self.payment_rate_manager = self.payment_rate_manager_class(
self.base_payment_rate_manager,
self.blob_tracker,
self.is_generous)
self.rate_limiter.start()
d1 = self.blob_manager.setup()
d2 = self.wallet.start()
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
dl.addCallback(lambda _: self.blob_tracker.start())
return dl
def setUp(self):
conf.initialize_settings(False)
self.blob_dir = tempfile.mkdtemp()
self.db_dir = tempfile.mkdtemp()
self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(self.db_dir))
self.peer = Peer('somehost', 22)
yield self.bm.storage.setup()
def setUp(self):
conf.initialize_settings()
self.blob_dir = tempfile.mkdtemp()
self.db_dir = tempfile.mkdtemp()
hash_announcer = DummyHashAnnouncer()
self.bm = DiskBlobManager(hash_announcer, self.blob_dir, self.db_dir)
self.peer = Peer('somehost', 22)
def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
raise Exception(
"TempBlobManager is no longer supported, specify BlobManager or db_dir"
)
else:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir,
self.storage)
if self.blob_tracker is None:
self.blob_tracker = self.blob_tracker_class(
self.blob_manager, self.peer_finder, self.dht_node)
if self.payment_rate_manager is None:
self.payment_rate_manager = self.payment_rate_manager_class(
self.base_payment_rate_manager, self.blob_tracker,
self.is_generous)
self.rate_limiter.start()
d = self.storage.setup()
d.addCallback(lambda _: self.wallet.start())
d.addCallback(lambda _: self.blob_tracker.start())
return d
def encrypt_blob(filename, key, iv):
dummy_announcer = DummyHashAnnouncer()
manager = DiskBlobManager(dummy_announcer, '.', '.')
yield manager.setup()
creator = CryptStreamCreator(manager, filename, key, iv_generator(iv))
with open(filename, 'r') as infile:
producer = FileBodyProducer(infile, readSize=2**22)
yield producer.startProducing(creator)
yield creator.stop()
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_dir)
try:
result = yield self.download_blob_from_peer(
peer, timeout, blob_hash, tmp_blob_manager)
finally:
yield tmp_blob_manager.stop()
yield threads.deferToThread(shutil.rmtree, tmp_dir)
defer.returnValue(result)
def download_it(peer, timeout, blob_hash):
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
storage = SQLiteStorage(tmp_dir, reactor)
yield storage.setup()
tmp_blob_manager = DiskBlobManager(tmp_dir, storage)
config = {'auto_connect': True}
if conf.settings['lbryum_wallet_dir']:
config['lbryum_wallet_dir'] = conf.settings['lbryum_wallet_dir']
config['use_keyring'] = False
config['blockchain_name'] = conf.settings['blockchain_name']
config['lbryum_servers'] = []
wallet = LbryWalletManager.from_lbrynet_config(config, storage)
downloader = SinglePeerDownloader()
downloader.setup(wallet)
try:
blob_downloaded = yield downloader.download_blob_from_peer(
peer, timeout, blob_hash, tmp_blob_manager)
if blob_downloaded:
log.info("SUCCESS!")
blob = yield tmp_blob_manager.get_blob(blob_hash)
pprint(blob)
if not blob.verified:
log.error("except that its not verified....")
else:
reader = BlobStreamDescriptorReader(blob)
info = None
for x in range(0, 3):
try:
info = yield reader.get_info()
except ValueError:
pass
if info:
break
# there's some kind of race condition where it sometimes doesnt write the blob to disk in time
time.sleep(0.1)
if info is not None:
pprint(info)
for content_blob in info['blobs']:
if 'blob_hash' in content_blob:
yield download_it(peer, timeout,
content_blob['blob_hash'])
else:
log.error("Download failed")
finally:
yield tmp_blob_manager.stop()
yield threads.deferToThread(shutil.rmtree, tmp_dir)
defer.returnValue(True)
def start(self):
if self.peer_manager is None:
self.peer_manager = PeerManager()
def match_port(h, p):
return h, p
def join_resolved_addresses(result):
addresses = []
for success, value in result:
if success is True:
addresses.append(value)
return addresses
def start_dht(addresses):
log.info("Starting the dht")
log.info("lbry id: %s",
base64.encodestring(self.lbryid).strip("\n"))
self.dht_node.joinNetwork(addresses)
self.peer_finder.run_manage_loop()
self.hash_announcer.run_manage_loop()
ds = []
for host, port in self.known_dht_nodes:
d = reactor.resolve(host)
d.addCallback(match_port, port)
ds.append(d)
if self.dht_node is None:
self.dht_node = Node(udpPort=self.dht_node_port,
lbryid=self.lbryid,
externalIP=self.external_ip)
if self.peer_finder is None:
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
if self.hash_announcer is None:
self.hash_announcer = DHTHashAnnouncer(self.dht_node,
self.peer_port)
if self.blob_manager is None:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir,
self.blob_data_dir)
d1 = defer.DeferredList(ds)
d1.addCallback(join_resolved_addresses)
d1.addCallback(start_dht)
d2 = self.blob_manager.setup()
dl = defer.DeferredList([d1, d2],
fireOnOneErrback=True,
consumeErrors=True)
return dl
def download_it(peer, timeout, blob_hash):
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
announcer = DummyHashAnnouncer()
tmp_blob_manager = DiskBlobManager(announcer, tmp_dir, tmp_dir)
config = {'auto_connect': True}
if conf.settings['lbryum_wallet_dir']:
config['lbryum_path'] = conf.settings['lbryum_wallet_dir']
storage = Wallet.InMemoryStorage()
wallet = Wallet.LBRYumWallet(storage, config)
downloader = SinglePeerDownloader()
downloader.setup(wallet)
try:
blob_downloaded = yield downloader.download_blob_from_peer(
peer, timeout, blob_hash, tmp_blob_manager)
if blob_downloaded:
log.info("SUCCESS!")
blob = yield tmp_blob_manager.get_blob(blob_hash)
pprint(blob)
if not blob.verified:
log.error("except that its not verified....")
else:
reader = BlobStreamDescriptorReader(blob)
info = None
for x in range(0, 3):
try:
info = yield reader.get_info()
except ValueError:
pass
if info:
break
time.sleep(
0.1
) # there's some kind of race condition where it sometimes doesnt write the blob to disk in time
if info is not None:
pprint(info)
for content_blob in info['blobs']:
if 'blob_hash' in content_blob:
yield download_it(peer, timeout,
content_blob['blob_hash'])
else:
log.error("Download failed")
finally:
yield tmp_blob_manager.stop()
yield threads.deferToThread(shutil.rmtree, tmp_dir)
defer.returnValue(True)
def setUp(self):
mocks.mock_conf_settings(self)
self.tmp_db_dir, self.tmp_blob_dir = mk_db_and_blob_dir()
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2)
self.rate_limiter = DummyRateLimiter()
self.sd_identifier = StreamDescriptorIdentifier()
self.storage = SQLiteStorage(self.tmp_db_dir)
self.blob_manager = DiskBlobManager(self.tmp_blob_dir, self.storage)
self.prm = OnlyFreePaymentsManager()
self.lbry_file_manager = EncryptedFileManager(
self.peer_finder, self.rate_limiter, self.blob_manager,
self.wallet, self.prm, self.storage, self.sd_identifier)
d = self.storage.setup()
d.addCallback(lambda _: self.lbry_file_manager.setup())
return d
def setup(self):
init_conf_windows()
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.rate_limiter = RateLimiter()
if self.ul_rate_limit is not None:
self.rate_limiter.set_ul_limit(self.ul_rate_limit)
self.prm = OnlyFreePaymentsManager()
self.storage = SQLiteStorage(self.db_dir)
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage)
self.lbry_file_manager = EncryptedFileManager(
FakePeerFinder(5553, self.peer_manager, 1), self.rate_limiter,
self.blob_manager, self.wallet, self.prm, self.storage,
StreamDescriptorIdentifier())
yield self.storage.setup()
yield self.blob_manager.setup()
yield self.lbry_file_manager.setup()
query_handler_factories = {
1:
BlobAvailabilityHandlerFactory(self.blob_manager),
2:
BlobRequestHandlerFactory(self.blob_manager, self.wallet, self.prm,
None),
3:
self.wallet.get_wallet_info_query_handler_factory(),
}
server_factory = ServerProtocolFactory(self.rate_limiter,
query_handler_factories,
self.peer_manager)
self.server_port = reactor.listenTCP(5553,
server_factory,
interface="localhost")
test_file = GenFile(self.file_size,
b''.join([chr(i) for i in xrange(0, 64, 6)]))
lbry_file = yield create_lbry_file(self.blob_manager, self.storage,
self.prm, self.lbry_file_manager,
"test_file", test_file)
defer.returnValue(lbry_file.sd_hash)
def setUp(self):
mocks.mock_conf_settings(self)
self.db_dir, self.blob_dir = mk_db_and_blob_dir()
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 1)
self.rate_limiter = RateLimiter()
self.prm = OnlyFreePaymentsManager()
self.storage = SQLiteStorage(self.db_dir)
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage)
self.sd_identifier = StreamDescriptorIdentifier()
self.lbry_file_manager = EncryptedFileManager(
self.peer_finder, self.rate_limiter, self.blob_manager,
self.wallet, self.prm, self.storage, self.sd_identifier)
self.uploader = LbryUploader(5209343)
self.sd_hash = yield self.uploader.setup()
yield self.storage.setup()
yield self.blob_manager.setup()
yield self.lbry_file_manager.setup()
yield add_lbry_file_to_sd_identifier(self.sd_identifier)
def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
self.blob_manager = TempBlobManager(self.hash_announcer)
else:
self.blob_manager = DiskBlobManager(self.hash_announcer,
self.blob_dir, self.db_dir)
self.rate_limiter.start()
d1 = self.blob_manager.setup()
d2 = self.wallet.start()
dl = defer.DeferredList([d1, d2],
fireOnOneErrback=True,
consumeErrors=True)
dl.addErrback(lambda err: err.value.subFailure)
return dl
def setUp(self):
mocks.mock_conf_settings(self)
self.session = None
self.lbry_file_manager = None
self.is_generous = True
self.db_dir = tempfile.mkdtemp()
self.blob_dir = os.path.join(self.db_dir, "blobfiles")
os.mkdir(self.blob_dir)
self.dht_node = FakeNode()
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2)
self.rate_limiter = DummyRateLimiter()
self.sd_identifier = StreamDescriptorIdentifier()
self.storage = SQLiteStorage(self.db_dir)
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage,
self.dht_node._dataStore)
self.prm = OnlyFreePaymentsManager()
self.lbry_file_manager = EncryptedFileManager(
self.peer_finder, self.rate_limiter, self.blob_manager,
self.wallet, self.prm, self.storage, self.sd_identifier)
d = self.storage.setup()
d.addCallback(lambda _: self.lbry_file_manager.setup())
return d
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage,
dht_node._dataStore)
return self.blob_manager.setup()
