def setUp(self):
self.loop = asyncio.get_event_loop()
self.peer_manager = PeerManager(self.loop)
self.node_ids = [generate_id(), generate_id(), generate_id()]
self.first_contact = self.peer_manager.get_kademlia_peer(
self.node_ids[1], '127.0.0.1', udp_port=1000)
self.second_contact = self.peer_manager.get_kademlia_peer(
self.node_ids[0], '192.168.0.1', udp_port=1000)
def setUp(self):
self.contact_manager = ContactManager()
self.node_ids = [generate_id(), generate_id(), generate_id()]
make_contact = self.contact_manager.make_contact
self.first_contact = make_contact(self.node_ids[1], '127.0.0.1', 1000,
None, 1)
self.second_contact = make_contact(self.node_ids[0], '192.168.0.1',
1000, None, 32)
self.second_contact_second_reference = make_contact(
self.node_ids[0], '192.168.0.1', 1000, None, 32)
self.first_contact_different_values = make_contact(
self.node_ids[1], '192.168.1.20', 1000, None, 50)
def testGetContacts(self):
# try and get 2 contacts from empty list
result = self.kbucket.getContacts(2)
self.assertFalse(len(result) != 0, "Returned list should be empty; returned list length: %d" %
(len(result)))
# Add k-2 contacts
node_ids = []
if constants.k >= 2:
for i in range(constants.k-2):
node_ids.append(generate_id())
tmpContact = self.contact_manager.make_contact(node_ids[-1], next(self.address_generator), 4444, 0,
None)
self.kbucket.addContact(tmpContact)
else:
# add k contacts
for i in range(constants.k):
node_ids.append(generate_id())
tmpContact = self.contact_manager.make_contact(node_ids[-1], next(self.address_generator), 4444, 0,
None)
self.kbucket.addContact(tmpContact)
# try to get too many contacts
# requested count greater than bucket size; should return at most k contacts
contacts = self.kbucket.getContacts(constants.k+3)
self.assertTrue(len(contacts) <= constants.k,
'Returned list should not have more than k entries!')
# verify returned contacts in list
for node_id, i in zip(node_ids, range(constants.k-2)):
self.assertFalse(self.kbucket._contacts[i].id != node_id,
"Contact in position %s not same as added contact" % (str(i)))
# try to get too many contacts
# requested count one greater than number of contacts
if constants.k >= 2:
result = self.kbucket.getContacts(constants.k-1)
self.assertFalse(len(result) != constants.k-2,
"Too many contacts in returned list %s - should be %s" %
(len(result), constants.k-2))
else:
result = self.kbucket.getContacts(constants.k-1)
# if the count is <= 0, it should return all of it's contats
self.assertFalse(len(result) != constants.k,
"Too many contacts in returned list %s - should be %s" %
(len(result), constants.k-2))
result = self.kbucket.getContacts(constants.k-3)
self.assertFalse(len(result) != constants.k-3,
"Too many contacts in returned list %s - should be %s" %
(len(result), constants.k-3))
def test_add_peer(self):
peer = KademliaPeer(None,
'1.2.3.4',
constants.generate_id(2),
udp_port=4444)
peer_update2 = KademliaPeer(None,
'1.2.3.4',
constants.generate_id(2),
udp_port=4445)
self.assertListEqual([], self.kbucket.peers)
# add the peer
self.kbucket.add_peer(peer)
self.assertListEqual([peer], self.kbucket.peers)
# re-add it
self.kbucket.add_peer(peer)
self.assertListEqual([peer], self.kbucket.peers)
self.assertEqual(self.kbucket.peers[0].udp_port, 4444)
# add a new peer object with the same id and address but a different port
self.kbucket.add_peer(peer_update2)
self.assertListEqual([peer_update2], self.kbucket.peers)
self.assertEqual(self.kbucket.peers[0].udp_port, 4445)
# modify the peer object to have a different port
peer_update2.udp_port = 4444
self.kbucket.add_peer(peer_update2)
self.assertListEqual([peer_update2], self.kbucket.peers)
self.assertEqual(self.kbucket.peers[0].udp_port, 4444)
self.kbucket.peers.clear()
# Test if contacts can be added to empty list
# Add k contacts to bucket
for i in range(constants.k):
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
self.assertEqual(peer, self.kbucket.peers[i])
# Test if contact is not added to full list
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertFalse(self.kbucket.add_peer(peer))
# Test if an existing contact is updated correctly if added again
existing_peer = self.kbucket.peers[0]
self.assertTrue(self.kbucket.add_peer(existing_peer))
self.assertEqual(existing_peer, self.kbucket.peers[-1])
def testRemoveContact(self):
# try remove contact from empty list
rmContact = self.contact_manager.make_contact(generate_id(), next(self.address_generator), 4444, 0, None)
self.assertRaises(ValueError, self.kbucket.removeContact, rmContact)
# Add couple contacts
for i in range(constants.k-2):
tmpContact = self.contact_manager.make_contact(generate_id(), next(self.address_generator), 4444, 0, None)
self.kbucket.addContact(tmpContact)
# try remove contact from empty list
self.kbucket.addContact(rmContact)
result = self.kbucket.removeContact(rmContact)
self.assertNotIn(rmContact, self.kbucket._contacts, "Could not remove contact from bucket")
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
file_path = os.path.join(self.server_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(os.urandom(20000000))
descriptor = await StreamDescriptor.create_stream(
self.loop, self.server_blob_manager.blob_dir, file_path, old_sort=old_sort
)
self.sd_hash = descriptor.sd_hash
self.mock_wallet, self.uri = get_mock_wallet(self.sd_hash, self.client_storage, balance, fee)
self.stream_manager = StreamManager(self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
self.client_storage, get_mock_node(self.server_from_client),
AnalyticsManager(self.client_config,
binascii.hexlify(generate_id()).decode(),
binascii.hexlify(generate_id()).decode()))
self.exchange_rate_manager = get_dummy_exchange_rate_manager(time)
def _generateID(self):
""" Generates an n-bit pseudo-random identifier
@return: A globally unique n-bit pseudo-random identifier
@rtype: str
"""
return generate_id()
def __init__(self,
loop: asyncio.BaseEventLoop,
blob_manager: 'BlobFileManager',
rowid: int,
descriptor: 'StreamDescriptor',
download_directory: str,
file_name: typing.Optional[str],
downloader: typing.Optional[StreamDownloader] = None,
status: typing.Optional[str] = STATUS_STOPPED,
claim: typing.Optional[StoredStreamClaim] = None,
download_id: typing.Optional[str] = None):
self.loop = loop
self.blob_manager = blob_manager
self.rowid = rowid
self.download_directory = download_directory
self._file_name = file_name
self.descriptor = descriptor
self.downloader = downloader
self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim
self._status = status
self.fully_reflected = asyncio.Event(loop=self.loop)
self.tx = None
self.download_id = download_id or binascii.hexlify(
generate_id()).decode()
def setUp(self):
self.clock = task.Clock()
self.contact_manager = ContactManager(self.clock.seconds)
self.contact = self.contact_manager.make_contact(
generate_id(), "127.0.0.1", 4444, None)
self.clock.advance(3600)
self.assertIsNone(self.contact.contact_is_good)
def start(self):
self.upnp_component = self.component_manager.get_component(
UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get(
"TCP", GCS("peer_port"))
self.external_udp_port = self.upnp_component.upnp_redirects.get(
"UDP", GCS("dht_node_port"))
node_id = CS.get_node_id()
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = yield CS.get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = Node(node_id=node_id,
udpPort=GCS('dht_node_port'),
externalUDPPort=self.external_udp_port,
externalIP=external_ip,
peerPort=self.external_peer_port)
yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False)
log.info("Started the dht")
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
descriptor: typing.Optional[StreamDescriptor] = None,
content_fee: typing.Optional['Transaction'] = None,
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.download_directory = download_directory
self._file_name = file_name
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.written_bytes = 0
self.content_fee = content_fee
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop)
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop: typing.Optional[asyncio.Handle] = None
self.saving = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=self.loop)
self.started_writing = asyncio.Event(loop=self.loop)
def test_add_peer(self):
# Test if contacts can be added to empty list
# Add k contacts to bucket
for i in range(constants.k):
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
self.assertEqual(peer, self.kbucket.peers[i])
# Test if contact is not added to full list
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertFalse(self.kbucket.add_peer(peer))
# Test if an existing contact is updated correctly if added again
existing_peer = self.kbucket.peers[0]
self.assertTrue(self.kbucket.add_peer(existing_peer))
self.assertEqual(existing_peer, self.kbucket.peers[-1])
def get_node_id(self):
node_id_filename = os.path.join(self.conf.data_dir, "node_id")
if os.path.isfile(node_id_filename):
with open(node_id_filename, "r") as node_id_file:
return base58.b58decode(str(node_id_file.read()).strip())
node_id = utils.generate_id()
with open(node_id_filename, "w") as node_id_file:
node_id_file.write(base58.b58encode(node_id).decode())
return node_id
def test_remove_peer(self):
# try remove contact from empty list
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertRaises(ValueError, self.kbucket.remove_peer, peer)
added = []
# Add couple contacts
for i in range(constants.k - 2):
peer = self.peer_manager.get_kademlia_peer(
generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
added.append(peer)
while added:
peer = added.pop()
self.assertIn(peer, self.kbucket.peers)
self.kbucket.remove_peer(peer)
self.assertNotIn(peer, self.kbucket.peers)
def test_get_contact(self):
""" Tests if a specific existing contact can be retrieved correctly """
contact_id = generate_id(b'node2')
contact = self.contact_manager.make_contact(contact_id, '127.0.0.1',
9182, self.protocol)
# Now add it...
yield self.routingTable.addContact(contact)
# ...and get it again
same_contact = self.routingTable.getContact(contact_id)
self.assertEqual(contact, same_contact,
'getContact() should return the same contact')
def get_node_id(self):
node_id_filename = os.path.join(self.ensure_data_dir(), "node_id")
if not self._node_id:
if os.path.isfile(node_id_filename):
with open(node_id_filename, "r") as node_id_file:
self._node_id = base58.b58decode(
str(node_id_file.read()).strip())
if not self._node_id:
self._node_id = utils.generate_id()
with open(node_id_filename, "w") as node_id_file:
node_id_file.write(base58.b58encode(self._node_id).decode())
return self._node_id
def test_add_contact(self):
""" Tests if a contact can be added and retrieved correctly """
# Create the contact
contact_id = generate_id(b'node2')
contact = self.contact_manager.make_contact(contact_id, '127.0.0.1',
9182, self.protocol)
# Now add it...
yield self.routingTable.addContact(contact)
# ...and request the closest nodes to it (will retrieve it)
closest_nodes = self.routingTable.findCloseNodes(contact_id)
self.assertEqual(len(closest_nodes), 1)
self.assertIn(contact, closest_nodes)
def get_installation_id(self):
install_id_filename = os.path.join(self.ensure_data_dir(),
"install_id")
if not self._installation_id:
if os.path.isfile(install_id_filename):
with open(install_id_filename, "r") as install_id_file:
self._installation_id = str(install_id_file.read()).strip()
if not self._installation_id:
self._installation_id = base58.b58encode(
utils.generate_id()).decode()
with open(install_id_filename, "w") as install_id_file:
install_id_file.write(self._installation_id)
return self._installation_id
def testAddContact(self):
""" Tests if the bucket handles contact additions/updates correctly """
# Test if contacts can be added to empty list
# Add k contacts to bucket
for i in range(constants.k):
tmpContact = self.contact_manager.make_contact(generate_id(), next(self.address_generator), 4444, 0, None)
self.kbucket.addContact(tmpContact)
self.assertEqual(
self.kbucket._contacts[i],
tmpContact,
"Contact in position %d not the same as the newly-added contact" % i)
# Test if contact is not added to full list
tmpContact = self.contact_manager.make_contact(generate_id(), next(self.address_generator), 4444, 0, None)
self.assertRaises(kbucket.BucketFull, self.kbucket.addContact, tmpContact)
# Test if an existing contact is updated correctly if added again
existingContact = self.kbucket._contacts[0]
self.kbucket.addContact(existingContact)
self.assertEqual(
self.kbucket._contacts.index(existingContact),
len(self.kbucket._contacts)-1,
'Contact not correctly updated; it should be at the end of the list of contacts')
def test_split_bucket(self):
""" Tests if the the routing table correctly dynamically splits k-buckets """
self.assertEqual(
self.routingTable._buckets[0].rangeMax, 2**384,
'Initial k-bucket range should be 0 <= range < 2**384')
# Add k contacts
for i in range(constants.k):
node_id = generate_id(b'remote node %d' % i)
contact = self.contact_manager.make_contact(
node_id, '127.0.0.1', 9182, self.protocol)
yield self.routingTable.addContact(contact)
self.assertEqual(
len(self.routingTable._buckets), 1,
'Only k nodes have been added; the first k-bucket should now '
'be full, but should not yet be split')
# Now add 1 more contact
node_id = generate_id(b'yet another remote node')
contact = self.contact_manager.make_contact(node_id, '127.0.0.1', 9182,
self.protocol)
yield self.routingTable.addContact(contact)
self.assertEqual(
len(self.routingTable._buckets), 2,
'k+1 nodes have been added; the first k-bucket should have been '
'split into two new buckets')
self.assertNotEqual(
self.routingTable._buckets[0].rangeMax, 2**384,
'K-bucket was split, but its range was not properly adjusted')
self.assertEqual(
self.routingTable._buckets[1].rangeMax, 2**384,
'K-bucket was split, but the second (new) bucket\'s '
'max range was not set properly')
self.assertEqual(
self.routingTable._buckets[0].rangeMax,
self.routingTable._buckets[1].rangeMin,
'K-bucket was split, but the min/max ranges were '
'not divided properly')
def __init__(self,
fixed_defaults,
adjustable_defaults,
persisted_settings=None,
environment=None,
cli_settings=None):
self._installation_id = None
self._session_id = base58.b58encode(utils.generate_id()).decode()
self._node_id = None
self._fixed_defaults = fixed_defaults
self._adjustable_defaults = adjustable_defaults
self._data = {
TYPE_DEFAULT: {}, # defaults
TYPE_PERSISTED:
{}, # stored settings from daemon_settings.yml (or from a db, etc)
TYPE_ENV: {}, # settings from environment variables
TYPE_CLI: {}, # command-line arguments
TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc)
}
# the order in which a piece of data is searched for. earlier types override later types
self._search_order = (TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED,
TYPE_DEFAULT)
# types of data where user specified config values can be stored
self._user_specified = (TYPE_RUNTIME, TYPE_CLI, TYPE_ENV,
TYPE_PERSISTED)
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
self._data[TYPE_DEFAULT].update(
{k: v[1]
for (k, v) in self._adjustable_defaults.items()})
if persisted_settings is None:
persisted_settings = {}
self._validate_settings(persisted_settings)
self._data[TYPE_PERSISTED].update(persisted_settings)
env_settings = self._parse_environment(environment)
self._validate_settings(env_settings)
self._data[TYPE_ENV].update(env_settings)
if cli_settings is None:
cli_settings = {}
self._validate_settings(cli_settings)
self._data[TYPE_CLI].update(cli_settings)
def test_remove_contact(self):
""" Tests contact removal """
# Create the contact
contact_id = generate_id(b'node2')
contact = self.contact_manager.make_contact(contact_id, '127.0.0.1',
9182, self.protocol)
# Now add it...
yield self.routingTable.addContact(contact)
# Verify addition
self.assertEqual(len(self.routingTable._buckets[0]), 1,
'Contact not added properly')
# Now remove it
self.routingTable.removeContact(contact)
self.assertEqual(len(self.routingTable._buckets[0]), 0,
'Contact not removed properly')
def test_store_and_expire(self):
blob_hash = generate_id(1)
announcing_node = self.nodes[20]
# announce the blob
announce_d = announcing_node.announceHaveBlob(blob_hash)
self.pump_clock(5+1)
storing_node_ids = yield announce_d
all_nodes = set(self.nodes).union(set(self._seeds))
# verify the nodes we think stored it did actually store it
storing_nodes = [node for node in all_nodes if hexlify(node.node_id) in storing_node_ids]
self.assertEqual(len(storing_nodes), len(storing_node_ids))
self.assertEqual(len(storing_nodes), constants.k)
for node in storing_nodes:
self.assertTrue(node._dataStore.hasPeersForBlob(blob_hash))
datastore_result = node._dataStore.getPeersForBlob(blob_hash)
self.assertEqual(list(map(lambda contact: (contact.id, contact.address, contact.port),
node._dataStore.getStoringContacts())), [(announcing_node.node_id,
announcing_node.externalIP,
announcing_node.port)])
self.assertEqual(len(datastore_result), 1)
expanded_peers = []
for peer in datastore_result:
host = ".".join([str(d) for d in peer[:4]])
port, = struct.unpack('>H', peer[4:6])
peer_node_id = peer[6:]
if (host, port, peer_node_id) not in expanded_peers:
expanded_peers.append((peer_node_id, host, port))
self.assertEqual(expanded_peers[0],
(announcing_node.node_id, announcing_node.externalIP, announcing_node.peerPort))
# verify the announced blob expires in the storing nodes datastores
self.clock.advance(constants.dataExpireTimeout) # skip the clock directly ahead
for node in storing_nodes:
self.assertFalse(node._dataStore.hasPeersForBlob(blob_hash))
datastore_result = node._dataStore.getPeersForBlob(blob_hash)
self.assertEqual(len(datastore_result), 0)
self.assertIn(blob_hash, node._dataStore) # the looping call shouldn't have removed it yet
self.assertEqual(len(node._dataStore.getStoringContacts()), 1)
self.pump_clock(constants.checkRefreshInterval + 1) # tick the clock forward (so the nodes refresh)
for node in storing_nodes:
self.assertFalse(node._dataStore.hasPeersForBlob(blob_hash))
datastore_result = node._dataStore.getPeersForBlob(blob_hash)
self.assertEqual(len(datastore_result), 0)
self.assertEqual(len(node._dataStore.getStoringContacts()), 0)
self.assertNotIn(blob_hash, node._dataStore.keys()) # the looping call should have fired
def test_boolean(self):
""" Test "equals" and "not equals" comparisons """
self.assertNotEqual(
self.first_contact,
self.contact_manager.make_contact(self.first_contact.id,
self.first_contact.address,
self.first_contact.port + 1,
None, 32))
self.assertNotEqual(
self.first_contact,
self.contact_manager.make_contact(self.first_contact.id,
'193.168.1.1',
self.first_contact.port, None,
32))
self.assertNotEqual(
self.first_contact,
self.contact_manager.make_contact(generate_id(),
self.first_contact.address,
self.first_contact.port, None,
32))
self.assertEqual(self.second_contact,
self.second_contact_second_reference)
def test_nullify_token(self):
blob_hash = generate_id(1)
announcing_node = self.nodes[20]
# announce the blob
announce_d = announcing_node.announceHaveBlob(blob_hash)
self.pump_clock(5+1)
storing_node_ids = yield announce_d
self.assertEqual(len(storing_node_ids), 8)
for node in set(self.nodes).union(set(self._seeds)):
# now, everyone has the wrong token
node.change_token()
node.change_token()
announce_d = announcing_node.announceHaveBlob(blob_hash)
self.pump_clock(5+1)
storing_node_ids = yield announce_d
self.assertEqual(len(storing_node_ids), 0) # can't store, wrong tokens, but they get nullified
announce_d = announcing_node.announceHaveBlob(blob_hash)
self.pump_clock(5+1)
storing_node_ids = yield announce_d
self.assertEqual(len(storing_node_ids), 8) # next attempt succeeds as it refreshes tokens
async def start(self):
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
node_id = conf.settings.get_node_id()
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = Node(
node_id=node_id,
udpPort=conf.settings['dht_node_port'],
externalUDPPort=self.external_udp_port,
externalIP=external_ip,
peerPort=self.external_peer_port
)
await d2f(self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False))
log.info("Started the dht")
def __init__(self,
fixed_defaults,
adjustable_defaults: typing.Dict,
persisted_settings=None,
environment=None,
cli_settings=None,
data_dir: optional_str = None,
wallet_dir: optional_str = None,
download_dir: optional_str = None,
file_name: optional_str = None):
self._installation_id = None
self._session_id = base58.b58encode(utils.generate_id()).decode()
self._node_id = None
self._fixed_defaults = fixed_defaults
# copy the default adjustable settings
self._adjustable_defaults = {
k: v
for k, v in adjustable_defaults.items()
}
# set the os specific default directories
if platform is WINDOWS:
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_windows_directories(
)
elif platform is DARWIN:
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_darwin_directories(
)
elif platform is LINUX:
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_linux_directories(
)
else:
assert None not in [data_dir, wallet_dir, download_dir]
if data_dir:
self.default_data_dir = data_dir
if wallet_dir:
self.default_wallet_dir = wallet_dir
if download_dir:
self.default_download_dir = download_dir
self._data = {
TYPE_DEFAULT: {}, # defaults
TYPE_PERSISTED:
{}, # stored settings from daemon_settings.yml (or from a db, etc)
TYPE_ENV: {}, # settings from environment variables
TYPE_CLI: {}, # command-line arguments
TYPE_RUNTIME: {}, # set during runtime (using self.set(), etc)
}
# the order in which a piece of data is searched for. earlier types override later types
self._search_order = (TYPE_RUNTIME, TYPE_CLI, TYPE_ENV, TYPE_PERSISTED,
TYPE_DEFAULT)
# types of data where user specified config values can be stored
self._user_specified = (TYPE_RUNTIME, TYPE_CLI, TYPE_ENV,
TYPE_PERSISTED)
self._data[TYPE_DEFAULT].update(self._fixed_defaults)
self._data[TYPE_DEFAULT].update(
{k: v[1]
for (k, v) in self._adjustable_defaults.items()})
if persisted_settings is None:
persisted_settings = {}
self._validate_settings(persisted_settings)
self._data[TYPE_PERSISTED].update(persisted_settings)
env_settings = self._parse_environment(environment)
self._validate_settings(env_settings)
self._data[TYPE_ENV].update(env_settings)
if cli_settings is None:
cli_settings = {}
self._validate_settings(cli_settings)
self._data[TYPE_CLI].update(cli_settings)
self.file_name = file_name or 'daemon_settings.yml'
async def _download_stream_from_uri(
self,
uri,
timeout: float,
exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError(
"cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(
outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.stream.has_fee:
fee_amount = round(
exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount),
5)
max_fee_amount = round(
exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC",
Decimal(self.config.max_key_fee['amount'])), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config,
self.blob_manager,
claim.stream.source.sd_hash,
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader,
download_id, outpoint, claim,
resolved, file_name)), timeout)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time(
) - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)),
fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time,
self.loop.time() - start_time, download_id,
parse_lbry_uri(uri).name, outpoint, None if not stream else
len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(
stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay
if not downloader else downloader.fixed_peers_delay,
claim.stream.source.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].length, time_to_first_bytes,
None if not error else error.__class__.__name__))
if error:
raise error
return stream
def __init__(self, nodeID, method, methodArgs, rpcID=None):
if rpcID is None:
rpcID = generate_id()[:constants.rpc_id_length]
super().__init__(rpcID, nodeID)
self.request = method
self.args = methodArgs
def setUp(self):
self.address_generator = address_generator()
self.contact_manager = ContactManager()
self.kbucket = kbucket.KBucket(0, 2**constants.key_bits, generate_id())
