class MyCommunity(Community):
community_id = community_id
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.request_cache = RequestCache()
def send_intro_request(self, target_addr):
packet = self.create_introduction_request(target_addr, new_style=self.network.is_new_style(target_addr))
cache = NumberCache(self.request_cache, 'intro-req', self.global_time)
cache.future = Future()
self.request_cache.add(cache)
self.endpoint.send(target_addr, packet)
return cache.future
def introduction_response_callback(self, peer, dist, payload):
if self.request_cache.has('intro-req', payload.identifier):
cache = self.request_cache.pop('intro-req', payload.identifier)
if not cache.future.done():
cache.future.set_result(payload)
class RemoteQueryCommunity(Community):
"""
Community for general purpose SELECT-like queries into remote Channels database
"""
def __init__(self,
my_peer,
endpoint,
network,
metadata_store,
settings=None,
**kwargs):
super().__init__(my_peer, endpoint, network=network, **kwargs)
self.settings = settings or RemoteQueryCommunitySettings()
self.mds = metadata_store
# This object stores requests for "select" queries that we sent to other hosts.
# We keep track of peers we actually requested for data so people can't randomly push spam at us.
# Also, this keeps track of hosts we responded to. There is a possibility that
# those hosts will push back updates at us, so we need to allow it.
self.request_cache = RequestCache()
self.add_message_handler(RemoteSelectPayload, self.on_remote_select)
self.add_message_handler(SelectResponsePayload,
self.on_remote_select_response)
def send_remote_select(self, peer, processing_callback=None, **kwargs):
request = SelectRequest(self.request_cache, hexlify(peer.mid), kwargs,
processing_callback)
self.request_cache.add(request)
self.logger.info(f"Select to {hexlify(peer.mid)} with ({kwargs})")
self.ez_send(
peer,
RemoteSelectPayload(request.number,
json.dumps(kwargs).encode('utf8')))
async def process_rpc_query(self, json_bytes: bytes):
"""
Retrieve the result of a database query from a third party, encoded as raw JSON bytes (through `dumps`).
:raises TypeError: if the JSON contains invalid keys.
:raises ValueError: if no JSON could be decoded.
:raises pony.orm.dbapiprovider.OperationalError: if an illegal query was performed.
"""
request_sanitized = sanitize_query(json.loads(json_bytes),
self.settings.max_response_size)
return await self.mds.MetadataNode.get_entries_threaded(
**request_sanitized)
def send_db_results(self, peer, request_payload_id, db_results):
index = 0
while index < len(db_results):
data, index = entries_to_chunk(db_results,
self.settings.maximum_payload_size,
start_index=index)
self.ez_send(peer, SelectResponsePayload(request_payload_id, data))
@lazy_wrapper(RemoteSelectPayload)
async def on_remote_select(self, peer, request_payload):
try:
db_results = await self.process_rpc_query(request_payload.json)
# When we send our response to a host, we open a window of opportunity
# for it to push back updates
if db_results and not self.request_cache.has(
hexlify(peer.mid), request_payload.id):
self.request_cache.add(
PushbackWindow(self.request_cache, hexlify(peer.mid),
request_payload.id))
self.send_db_results(peer, request_payload.id, db_results)
except (OperationalError, TypeError, ValueError) as error:
self.logger.error(f"Remote select. The error occurred: {error}")
@lazy_wrapper(SelectResponsePayload)
async def on_remote_select_response(self, peer, response_payload):
"""
Match the the response that we received from the network to a query cache
and process it by adding the corresponding entries to the MetadataStore database.
This processes both direct responses and pushback (updates) responses
"""
self.logger.info(f"Response from {hexlify(peer.mid)}")
# ACHTUNG! the returned request cache can be either a SelectRequest or PushbackWindow
request = self.request_cache.get(hexlify(peer.mid),
response_payload.id)
if request is None:
return
# Check for limit on the number of packets per request
if request.packets_limit > 1:
request.packets_limit -= 1
else:
self.request_cache.pop(hexlify(peer.mid), response_payload.id)
processing_results = await self.mds.process_compressed_mdblob_threaded(
response_payload.raw_blob)
self.logger.info(f"Response result: {processing_results}")
# If we now about updated versions of the received stuff, push the updates back
if isinstance(
request,
SelectRequest) and self.settings.push_updates_back_enabled:
newer_entities = [
md for md, result in processing_results
if result == GOT_NEWER_VERSION
]
self.send_db_results(peer, response_payload.id, newer_entities)
# Query back the sender for preview contents for the new channels
# TODO: maybe transform this into a processing_callback?
if self.settings.channel_query_back_enabled:
new_channels = [
md for md, result in processing_results
if result in (UNKNOWN_CHANNEL, UNKNOWN_COLLECTION)
]
for channel in new_channels:
request_dict = {
"channel_pk": hexlify(channel.public_key),
"origin_id": channel.id_,
"first": 0,
"last": self.settings.max_channel_query_back,
}
self.send_remote_select(peer=peer, **request_dict)
if isinstance(request, SelectRequest) and request.processing_callback:
request.processing_callback(request, processing_results)
async def unload(self):
await self.request_cache.shutdown()
await super().unload()
class RemoteQueryCommunity(TriblerCommunity, EVAProtocolMixin):
"""
Community for general purpose SELECT-like queries into remote Channels database
"""
def __init__(self,
my_peer,
endpoint,
network,
rqc_settings: RemoteQueryCommunitySettings = None,
metadata_store=None,
**kwargs):
super().__init__(my_peer, endpoint, network=network, **kwargs)
self.rqc_settings = rqc_settings
self.mds: MetadataStore = metadata_store
# This object stores requests for "select" queries that we sent to other hosts.
# We keep track of peers we actually requested for data so people can't randomly push spam at us.
# Also, this keeps track of hosts we responded to. There is a possibility that
# those hosts will push back updates at us, so we need to allow it.
self.request_cache = RequestCache()
self.add_message_handler(RemoteSelectPayload, self.on_remote_select)
self.add_message_handler(RemoteSelectPayloadEva,
self.on_remote_select_eva)
self.add_message_handler(SelectResponsePayload,
self.on_remote_select_response)
self.eva_init()
self.eva_register_receive_callback(self.on_receive)
self.eva_register_send_complete_callback(self.on_send_complete)
self.eva_register_error_callback(self.on_error)
def on_receive(self, peer, binary_info, binary_data, nonce):
self.logger.debug(
f"EVA data received: peer {hexlify(peer.mid)}, info {binary_info}")
packet = (peer.address, binary_data)
self.on_packet(packet)
def on_send_complete(self, peer, binary_info, binary_data, nonce):
self.logger.debug(
f"EVA outgoing transfer complete: peer {hexlify(peer.mid)}, info {binary_info}"
)
def on_error(self, peer, exception):
self.logger.warning(
f"EVA transfer error: peer {hexlify(peer.mid)}, exception: {exception}"
)
def send_remote_select(self,
peer,
processing_callback=None,
force_eva_response=False,
**kwargs):
request_class = EvaSelectRequest if force_eva_response else SelectRequest
request = request_class(
self.request_cache,
hexlify(peer.mid),
kwargs,
peer,
processing_callback=processing_callback,
timeout_callback=self._on_query_timeout,
)
self.request_cache.add(request)
self.logger.debug(f"Select to {hexlify(peer.mid)} with ({kwargs})")
args = (request.number, convert_to_json(kwargs).encode('utf8'))
if force_eva_response:
self.ez_send(peer, RemoteSelectPayloadEva(*args))
else:
self.ez_send(peer, RemoteSelectPayload(*args))
return request
async def process_rpc_query(self, json_bytes: bytes):
"""
Retrieve the result of a database query from a third party, encoded as raw JSON bytes (through `dumps`).
:raises TypeError: if the JSON contains invalid keys.
:raises ValueError: if no JSON could be decoded.
:raises pony.orm.dbapiprovider.OperationalError: if an illegal query was performed.
"""
request_sanitized = sanitize_query(json.loads(json_bytes),
self.rqc_settings.max_response_size)
return await self.mds.get_entries_threaded(**request_sanitized)
def send_db_results(self,
peer,
request_payload_id,
db_results,
force_eva_response=False):
# Special case of empty results list - sending empty lz4 archive
if len(db_results) == 0:
self.ez_send(
peer,
SelectResponsePayload(request_payload_id, LZ4_EMPTY_ARCHIVE))
return
index = 0
while index < len(db_results):
transfer_size = (self.eva_protocol.binary_size_limit
if force_eva_response else
self.rqc_settings.maximum_payload_size)
data, index = entries_to_chunk(db_results,
transfer_size,
start_index=index,
include_health=True)
payload = SelectResponsePayload(request_payload_id, data)
if force_eva_response or (len(data) >
self.rqc_settings.maximum_payload_size):
self.eva_send_binary(peer, struct.pack('>i',
request_payload_id),
self.ezr_pack(payload.msg_id, payload))
else:
self.ez_send(peer, payload)
@lazy_wrapper(RemoteSelectPayloadEva)
async def on_remote_select_eva(self, peer, request_payload):
await self._on_remote_select_basic(peer,
request_payload,
force_eva_response=True)
@lazy_wrapper(RemoteSelectPayload)
async def on_remote_select(self, peer, request_payload):
await self._on_remote_select_basic(peer, request_payload)
async def _on_remote_select_basic(self,
peer,
request_payload,
force_eva_response=False):
try:
db_results = await self.process_rpc_query(request_payload.json)
# When we send our response to a host, we open a window of opportunity
# for it to push back updates
if db_results and not self.request_cache.has(
hexlify(peer.mid), request_payload.id):
self.request_cache.add(
PushbackWindow(self.request_cache, hexlify(peer.mid),
request_payload.id))
self.send_db_results(peer, request_payload.id, db_results,
force_eva_response)
except (OperationalError, TypeError, ValueError) as error:
self.logger.error(f"Remote select. The error occurred: {error}")
@lazy_wrapper(SelectResponsePayload)
async def on_remote_select_response(self, peer, response_payload):
"""
Match the the response that we received from the network to a query cache
and process it by adding the corresponding entries to the MetadataStore database.
This processes both direct responses and pushback (updates) responses
"""
self.logger.debug(f"Response from {hexlify(peer.mid)}")
# ACHTUNG! the returned request cache can be any one of SelectRequest, PushbackWindow
request = self.request_cache.get(hexlify(peer.mid),
response_payload.id)
if request is None:
return
# Check for limit on the number of packets per request
if request.packets_limit > 1:
request.packets_limit -= 1
else:
self.request_cache.pop(hexlify(peer.mid), response_payload.id)
processing_results = await self.mds.process_compressed_mdblob_threaded(
response_payload.raw_blob)
self.logger.debug(f"Response result: {processing_results}")
if isinstance(
request,
EvaSelectRequest) and not request.processing_results.done():
request.processing_results.set_result(processing_results)
# If we know about updated versions of the received stuff, push the updates back
if isinstance(
request,
SelectRequest) and self.rqc_settings.push_updates_back_enabled:
newer_entities = [
r.md_obj for r in processing_results
if r.obj_state == ObjState.LOCAL_VERSION_NEWER
]
self.send_db_results(peer, response_payload.id, newer_entities)
if self.rqc_settings.channel_query_back_enabled:
for result in processing_results:
# Query back the sender for preview contents for the new channels
# The fact that the object is previously unknown is indicated by process_payload in the
# .obj_state property of returned ProcessingResults objects.
if result.obj_state == ObjState.NEW_OBJECT and result.md_obj.metadata_type in (
CHANNEL_TORRENT,
COLLECTION_NODE,
):
request_dict = {
"metadata_type": [COLLECTION_NODE, REGULAR_TORRENT],
"channel_pk": result.md_obj.public_key,
"origin_id": result.md_obj.id_,
"first": 0,
"last": self.rqc_settings.max_channel_query_back,
}
self.send_remote_select(peer=peer, **request_dict)
# Query back for missing dependencies, e.g. thumbnail/description.
# The fact that some dependency is missing is checked by the lower layer during
# the query to process_payload and indicated through .missing_deps property of the
# ProcessingResults objects returned by process_payload.
for dep_query_dict in result.missing_deps:
self.send_remote_select(peer=peer, **dep_query_dict)
if isinstance(request, SelectRequest) and request.processing_callback:
request.processing_callback(request, processing_results)
# Remember that at least a single packet was received was received from the queried peer.
if isinstance(request, SelectRequest):
request.peer_responded = True
def _on_query_timeout(self, request_cache):
if not request_cache.peer_responded:
self.logger.debug(
"Remote query timeout, deleting peer: %s %s %s",
str(request_cache.peer.address),
hexlify(request_cache.peer.mid),
str(request_cache.request_kwargs),
)
self.network.remove_peer(request_cache.peer)
async def unload(self):
await self.request_cache.shutdown()
await super().unload()
class MyCommunity(Community):
master_peer = Peer(ECCrypto().generate_key(u"medium"))
def __init__(self, *args, **kwargs):
super(MyCommunity, self).__init__(*args, **kwargs)
self.request_cache = RequestCache()
def unload(self):
self.request_cache.shutdown()
super(MyCommunity, self).unload()
def finish_ping(self, cache, include=True):
global RESULTS
print(cache.hostname, cache.address, time.time() - cache.starttime)
if include:
if (cache.hostname, cache.address) in RESULTS:
RESULTS[(cache.hostname,
cache.address)].append(time.time() - cache.starttime)
else:
RESULTS[(cache.hostname,
cache.address)] = [time.time() - cache.starttime]
elif (cache.hostname, cache.address) not in RESULTS:
RESULTS[(cache.hostname, cache.address)] = []
self.next_ping()
def next_ping(self):
global CHECK_QUEUE
if CHECK_QUEUE:
hostname, address = CHECK_QUEUE.pop()
packet = self.create_introduction_request(address)
self.request_cache.add(
PingCache(self, hostname, address, time.time()))
self.endpoint.send(address, packet)
else:
reactor.callFromThread(reactor.stop)
def introduction_response_callback(self, peer, dist, payload):
if self.request_cache.has(u"introping", payload.identifier):
cache = self.request_cache.pop(u"introping", payload.identifier)
self.finish_ping(cache)
def started(self):
global CHECK_QUEUE
dnsmap = {}
for (address, port) in _DNS_ADDRESSES:
try:
ip = gethostbyname(address)
dnsmap[(ip, port)] = address
except:
pass
UNKNOWN_NAME = '*'
for (ip, port) in _DEFAULT_ADDRESSES:
hostname = dnsmap.get((ip, port), None)
if not hostname:
hostname = UNKNOWN_NAME
UNKNOWN_NAME = UNKNOWN_NAME + '*'
CHECK_QUEUE.append((hostname, (ip, port)))
CHECK_QUEUE = CHECK_QUEUE * CONST_REQUESTS
self.next_ping()
class TrustChainCommunity(Community):
"""
Community for reputation based on TrustChain tamper proof interaction history.
"""
community_id = unhexlify('5ad767b05ae592a02488272ca2a86b847d4562e1')
UNIVERSAL_BLOCK_LISTENER = b'UNIVERSAL_BLOCK_LISTENER'
DB_CLASS = TrustChainDB
DB_NAME = 'trustchain'
version = b'\x02'
def __init__(self, *args, **kwargs):
working_directory = kwargs.pop('working_directory', '')
self.persistence = kwargs.pop('persistence', None)
db_name = kwargs.pop('db_name', self.DB_NAME)
self.settings = kwargs.pop('settings', TrustChainSettings())
self.receive_block_lock = RLock()
super(TrustChainCommunity, self).__init__(*args, **kwargs)
self.request_cache = RequestCache()
self.logger = logging.getLogger(self.__class__.__name__)
if not self.persistence:
db_path = os.path.join(
working_directory, db_name
) if working_directory != ":memory:" else working_directory
self.persistence = self.DB_CLASS(
db_path, self.my_peer.public_key.key_to_bin())
self.relayed_broadcasts = set()
self.relayed_broadcasts_order = deque()
self.logger.debug(
"The trustchain community started with Public Key: %s",
hexlify(self.my_peer.public_key.key_to_bin()))
self.shutting_down = False
self.listeners_map = {} # Map of block_type -> [callbacks]
self.register_task("db_cleanup", self.do_db_cleanup, interval=600)
self.add_message_handler(HalfBlockPayload, self.received_half_block)
self.add_message_handler(CrawlRequestPayload,
self.received_crawl_request)
self.add_message_handler(CrawlResponsePayload,
self.received_crawl_response)
self.add_message_handler(HalfBlockPairPayload,
self.received_half_block_pair)
self.add_message_handler(HalfBlockBroadcastPayload,
self.received_half_block_broadcast)
self.add_message_handler(HalfBlockPairBroadcastPayload,
self.received_half_block_pair_broadcast)
self.add_message_handler(EmptyCrawlResponsePayload,
self.received_empty_crawl_response)
def do_db_cleanup(self):
"""
Cleanup the database if necessary.
"""
blocks_in_db = self.persistence.get_number_of_known_blocks()
if blocks_in_db > self.settings.max_db_blocks:
my_pk = self.my_peer.public_key.key_to_bin()
self.persistence.remove_old_blocks(
blocks_in_db - self.settings.max_db_blocks, my_pk)
def add_listener(self, listener, block_types):
"""
Add a listener for specific block types.
"""
for block_type in block_types:
if block_type not in self.listeners_map:
self.listeners_map[block_type] = []
self.listeners_map[block_type].append(listener)
self.persistence.block_types[block_type] = listener.BLOCK_CLASS
def remove_listener(self, listener, block_types):
for block_type in block_types:
if block_type in self.listeners_map and listener in self.listeners_map[
block_type]:
self.listeners_map[block_type].remove(listener)
if block_type in self.persistence.block_types:
self.persistence.block_types.pop(block_type, None)
def get_block_class(self, block_type):
"""
Get the block class for a specific block type.
"""
if block_type not in self.listeners_map or not self.listeners_map[
block_type]:
return TrustChainBlock
return self.listeners_map[block_type][0].BLOCK_CLASS
async def should_sign(self, block):
"""
Return whether we should sign the block in the passed message.
@param block: the block we want to sign or not.
"""
if block.type not in self.listeners_map:
return False # There are no listeners for this block
for listener in self.listeners_map[block.type]:
should_sign = await maybe_coroutine(listener.should_sign, block)
if should_sign:
return True
return False
def on_counter_signed(self, block):
"""
We just counter signed a block. Inform the listeners.
"""
if block.type in self.listeners_map:
for listener in self.listeners_map[block.type]:
listener.on_counter_signed_block(block)
def get_counter_tx(self, block):
"""
Return some counter tx content.
"""
if block.type not in self.listeners_map:
return block.transaction # There are no listeners for this block
if len(self.listeners_map[block.type]) > 1:
self.logger.warning(
"There should only be one listener when returning a counter transaction!"
)
return block.transaction
listener = self.listeners_map[block.type][0]
if not hasattr(listener, "get_counter_tx"):
return block.transaction
return listener.get_counter_tx(block)
def _add_broadcasted_blockid(self, block_id):
self.relayed_broadcasts.add(block_id)
self.relayed_broadcasts_order.append(block_id)
if len(self.relayed_broadcasts) > self.settings.broadcast_history_size:
to_remove = self.relayed_broadcasts_order.popleft()
self.relayed_broadcasts.remove(to_remove)
def send_block(self, block, address=None, ttl=1):
"""
Send a block to a specific address, or do a broadcast to known peers if no peer is specified.
"""
global_time = self.claim_global_time()
dist = GlobalTimeDistributionPayload(global_time)
if address:
self.logger.debug("Sending block to (%s:%d) (%s)", address[0],
address[1], block)
payload = HalfBlockPayload.from_half_block(block)
packet = self._ez_pack(self._prefix, 1, [dist, payload], False)
self.endpoint.send(address, packet)
else:
self.logger.debug("Broadcasting block %s", block)
payload = HalfBlockBroadcastPayload.from_half_block(block, ttl)
packet = self._ez_pack(self._prefix, 5, [dist, payload], False)
peers = self.get_peers()
for peer in random.sample(
peers, min(len(peers), self.settings.broadcast_fanout)):
self.endpoint.send(peer.address, packet)
self._add_broadcasted_blockid(block.block_id)
def send_block_pair(self, block1, block2, address=None, ttl=1):
"""
Send a half block pair to a specific address, or do a broadcast to known peers if no peer is specified.
"""
global_time = self.claim_global_time()
dist = GlobalTimeDistributionPayload(global_time)
if address:
self.logger.debug("Sending block pair to (%s:%d) (%s and %s)",
address[0], address[1], block1, block2)
payload = HalfBlockPairPayload.from_half_blocks(block1, block2)
packet = self._ez_pack(self._prefix, 4, [dist, payload], False)
self.endpoint.send(address, packet)
else:
self.logger.debug("Broadcasting blocks %s and %s", block1, block2)
payload = HalfBlockPairBroadcastPayload.from_half_blocks(
block1, block2, ttl)
packet = self._ez_pack(self._prefix, 6, [dist, payload], False)
peers = self.get_peers()
for peer in random.sample(
peers, min(len(peers), self.settings.broadcast_fanout)):
self.endpoint.send(peer.address, packet)
self._add_broadcasted_blockid(block1.block_id)
def self_sign_block(self, block_type=b'unknown', transaction=None):
return self.sign_block(self.my_peer,
block_type=block_type,
transaction=transaction)
def create_source_block(self, block_type=b'unknown', transaction=None):
"""
Create a source block without any initial counterparty to sign.
:param block_type: The type of the block to be constructed, as a string
:param transaction: A string describing the interaction in this block
:return: A future that fires with a (block, None) tuple
"""
return self.sign_block(peer=None,
public_key=ANY_COUNTERPARTY_PK,
block_type=block_type,
transaction=transaction)
def create_link(self,
source,
block_type,
additional_info=None,
public_key=None):
"""
Create a Link Block to a source block
:param source: The source block which had no initial counterpary to sign
:param block_type: The type of the block to be constructed, as a string
:param additional_info: a dictionary with supplementary information concerning the transaction
:param public_key: The public key of the counterparty (usually of the source's owner)
:return: None
"""
public_key = source.public_key if public_key is None else public_key
return self.sign_block(self.my_peer,
linked=source,
public_key=public_key,
block_type=block_type,
additional_info=additional_info)
@synchronized
def sign_block(self,
peer,
public_key=EMPTY_PK,
block_type=b'unknown',
transaction=None,
linked=None,
additional_info=None):
"""
Create, sign, persist and send a block signed message
:param peer: The peer with whom you have interacted, as a IPv8 peer
:param public_key: The public key of the other party you transact with
:param block_type: The type of the block to be constructed, as a string
:param transaction: A string describing the interaction in this block
:param linked: The block that the requester is asking us to sign
:param additional_info: Stores additional information, on the transaction
"""
# NOTE to the future: This method reads from the database, increments and then writes back. If in some future
# this method is allowed to execute in parallel, be sure to lock from before .create up to after .add_block
# In this particular case there must be an implicit transaction due to the following assert
assert peer is not None or peer is None and linked is None and public_key == ANY_COUNTERPARTY_PK, \
"Peer, linked block should not be provided when creating a no counterparty source block. Public key " \
"should be that reserved for any counterpary."
assert transaction is None and linked is not None or transaction is not None and linked is None, \
"Either provide a linked block or a transaction, not both %s, %s" % (peer, self.my_peer)
assert (additional_info is None or additional_info is not None and linked is not None
and transaction is None), \
"Either no additional info is provided or one provides it for a linked block"
assert (linked is None or linked.link_public_key
== self.my_peer.public_key.key_to_bin()
or linked.link_public_key == ANY_COUNTERPARTY_PK
), "Cannot counter sign block not addressed to self"
assert linked is None or linked.link_sequence_number == UNKNOWN_SEQ, \
"Cannot counter sign block that is not a request"
assert transaction is None or isinstance(
transaction, dict), "Transaction should be a dictionary"
assert additional_info is None or isinstance(
additional_info, dict), "Additional info should be a dictionary"
self.persistence_integrity_check()
if linked and linked.link_public_key != ANY_COUNTERPARTY_PK:
block_type = linked.type
block = self.get_block_class(block_type).create(
block_type,
transaction,
self.persistence,
self.my_peer.public_key.key_to_bin(),
link=linked,
additional_info=additional_info,
link_pk=public_key)
block.sign(self.my_peer.key)
validation = block.validate(self.persistence)
self.logger.info("Signed block to %s (%s) validation result %s",
hexlify(block.link_public_key)[-8:], block,
validation)
if validation[0] != ValidationResult.partial_next and validation[
0] != ValidationResult.valid:
self.logger.error("Signed block did not validate?! Result %s",
repr(validation))
return fail(RuntimeError("Signed block did not validate."))
if not self.persistence.contains(block):
self.persistence.add_block(block)
self.notify_listeners(block)
# This is a source block with no counterparty
if not peer and public_key == ANY_COUNTERPARTY_PK:
if block.type not in self.settings.block_types_bc_disabled:
self.send_block(block)
return succeed((block, None))
# If there is a counterparty to sign, we send it
self.send_block(block, address=peer.address)
# We broadcast the block in the network if we initiated a transaction
if block.type not in self.settings.block_types_bc_disabled and not linked:
self.send_block(block)
if peer == self.my_peer:
# We created a self-signed block
if block.type not in self.settings.block_types_bc_disabled:
self.send_block(block)
return succeed(
(block,
None)) if public_key == ANY_COUNTERPARTY_PK else succeed(
(block, linked))
elif not linked:
# We keep track of this outstanding sign request.
sign_future = Future()
self.request_cache.add(
HalfBlockSignCache(self, block, sign_future, peer.address))
return sign_future
else:
# We return a future that fires immediately with both half blocks.
if block.type not in self.settings.block_types_bc_disabled:
self.send_block_pair(linked, block)
return succeed((linked, block))
@synchronized
@lazy_wrapper_unsigned(GlobalTimeDistributionPayload, HalfBlockPayload)
async def received_half_block(self, source_address, dist, payload):
"""
We've received a half block, either because we sent a SIGNED message to some one or we are crawling
"""
peer = Peer(payload.public_key, source_address)
block = self.get_block_class(payload.type).from_payload(
payload, self.serializer)
try:
await self.process_half_block(block, peer)
except RuntimeError as e:
self.logger.info("Failed to process half block (error %s)", e)
@synchronized
@lazy_wrapper_unsigned(GlobalTimeDistributionPayload,
HalfBlockBroadcastPayload)
def received_half_block_broadcast(self, source_address, dist, payload):
"""
We received a half block, part of a broadcast. Disseminate it further.
"""
block = self.get_block_class(payload.type).from_payload(
payload, self.serializer)
self.validate_persist_block(block)
if block.block_id not in self.relayed_broadcasts and payload.ttl > 1:
self.send_block(block, ttl=payload.ttl - 1)
@synchronized
@lazy_wrapper_unsigned(GlobalTimeDistributionPayload, HalfBlockPairPayload)
def received_half_block_pair(self, source_address, dist, payload):
"""
We received a block pair message.
"""
block1, block2 = self.get_block_class(payload.type1).from_pair_payload(
payload, self.serializer)
self.validate_persist_block(block1)
self.validate_persist_block(block2)
@synchronized
@lazy_wrapper_unsigned(GlobalTimeDistributionPayload,
HalfBlockPairBroadcastPayload)
def received_half_block_pair_broadcast(self, source_address, dist,
payload):
"""
We received a half block pair, part of a broadcast. Disseminate it further.
"""
block1, block2 = self.get_block_class(payload.type1).from_pair_payload(
payload, self.serializer)
self.validate_persist_block(block1)
self.validate_persist_block(block2)
if block1.block_id not in self.relayed_broadcasts and payload.ttl > 1:
self.send_block_pair(block1, block2, ttl=payload.ttl - 1)
def validate_persist_block(self, block):
"""
Validate a block and if it's valid, persist it. Return the validation result.
:param block: The block to validate and persist.
:return: [ValidationResult]
"""
validation = block.validate(self.persistence)
if validation[0] == ValidationResult.invalid:
pass
elif not self.persistence.contains(block):
self.persistence.add_block(block)
self.notify_listeners(block)
return validation
def notify_listeners(self, block):
"""
Notify listeners of a specific new block.
"""
# Call the listeners associated to the universal block, if there are any
for listener in self.listeners_map.get(self.UNIVERSAL_BLOCK_LISTENER,
[]):
listener.received_block(block)
# Avoid proceeding any further if the type of the block coincides with the UNIVERSAL_BLOCK_LISTENER
if block.type not in self.listeners_map or self.shutting_down or block.type == self.UNIVERSAL_BLOCK_LISTENER:
return
for listener in self.listeners_map[block.type]:
listener.received_block(block)
@synchronized
async def process_half_block(self, blk, peer):
"""
Process a received half block.
"""
validation = self.validate_persist_block(blk)
self.logger.info("Block validation result %s, %s, (%s)", validation[0],
validation[1], blk)
if validation[0] == ValidationResult.invalid:
raise RuntimeError(
f"Block could not be validated: {validation[0]}, {validation[1]}"
)
# Check if we are waiting for this signature response
link_block_id_int = int(hexlify(blk.linked_block_id), 16) % 100000000
if self.request_cache.has('sign', link_block_id_int):
cache = self.request_cache.pop('sign', link_block_id_int)
cache.sign_future.set_result(
(blk, self.persistence.get_linked(blk)))
# Is this a request, addressed to us, and have we not signed it already?
if (blk.link_sequence_number != UNKNOWN_SEQ
or blk.link_public_key != self.my_peer.public_key.key_to_bin()
or self.persistence.get_linked(blk) is not None):
return
self.logger.info("Received request block addressed to us (%s)", blk)
try:
should_sign = await maybe_coroutine(self.should_sign, blk)
except Exception as e:
self.logger.error(
"Error while determining whether to sign (error: %s)", e)
return
if not should_sign:
self.logger.info("Not signing block %s", blk)
return
# It is important that the request matches up with its previous block, gaps cannot be tolerated at
# this point. We already dropped invalids, so here we delay this message if the result is partial,
# partial_previous or no-info. We send a crawl request to the requester to (hopefully) close the gap
if (validation[0] == ValidationResult.partial_previous
or validation[0] == ValidationResult.partial
or validation[0] == ValidationResult.no_info
) and self.settings.validation_range > 0:
self.logger.info(
"Request block could not be validated sufficiently, crawling requester. %s",
validation)
# Note that this code does not cover the scenario where we obtain this block indirectly.
if not self.request_cache.has("crawl", blk.hash_number):
try:
await self.send_crawl_request(
peer,
blk.public_key,
max(GENESIS_SEQ, (blk.sequence_number -
self.settings.validation_range)),
max(GENESIS_SEQ, blk.sequence_number - 1),
for_half_block=blk)
except Exception as e:
self.logger.error(
"Error while sending crawl request (error: %s)", e)
return
return await self.process_half_block(blk, peer)
else:
blocks = self.sign_block(peer,
linked=blk,
additional_info=self.get_counter_tx(blk))
self.on_counter_signed(blk)
return blocks
def crawl_chain(self, peer, latest_block_num=0):
"""
Crawl the whole chain of a specific peer.
:param latest_block_num: The latest block number of the peer in question, if available.
"""
if self.request_cache.has("chaincrawl",
ChainCrawlCache.get_number_for(peer)):
self.logger.debug(
"Skipping crawl of peer %s, another crawl is pending", peer)
return succeed(None)
crawl_future = Future()
cache = ChainCrawlCache(self,
peer,
crawl_future,
known_chain_length=latest_block_num)
self.request_cache.add(cache)
ensure_future(self.send_next_partial_chain_crawl_request(cache))
return crawl_future
def crawl_lowest_unknown(self, peer, latest_block_num=None):
"""
Crawl the lowest unknown block of a specific peer.
:param latest_block_num: The latest block number of the peer in question, if available
"""
sq = self.persistence.get_lowest_sequence_number_unknown(
peer.public_key.key_to_bin())
if latest_block_num and sq == latest_block_num + 1:
return [
] # We don't have to crawl this node since we have its whole chain
return self.send_crawl_request(peer, peer.public_key.key_to_bin(), sq,
sq)
def send_crawl_request(self,
peer,
public_key,
start_seq_num,
end_seq_num,
for_half_block=None):
"""
Send a crawl request to a specific peer.
"""
crawl_id = for_half_block.hash_number if for_half_block else \
RandomNumberCache.find_unclaimed_identifier(self.request_cache, "crawl")
crawl_future = Future()
self.request_cache.add(CrawlRequestCache(self, crawl_id, crawl_future))
self.logger.info(
"Requesting crawl of node %s (blocks %d to %d) with id %d",
hexlify(peer.public_key.key_to_bin())[-8:], start_seq_num,
end_seq_num, crawl_id)
global_time = self.claim_global_time()
auth = BinMemberAuthenticationPayload(
self.my_peer.public_key.key_to_bin())
payload = CrawlRequestPayload(public_key, start_seq_num, end_seq_num,
crawl_id)
dist = GlobalTimeDistributionPayload(global_time)
packet = self._ez_pack(self._prefix, 2, [auth, dist, payload])
self.endpoint.send(peer.address, packet)
return crawl_future
@task
async def perform_partial_chain_crawl(self, cache, start, stop):
"""
Perform a partial crawl request for a specific range, when crawling a chain.
:param cache: The cache that stores progress regarding the chain crawl.
:param start: The sequence number of the first block to be requested.
:param stop: The sequence number of the last block to be requested.
"""
if cache.current_request_range != (start, stop):
# We are performing a new request
cache.current_request_range = start, stop
cache.current_request_attempts = 0
elif cache.current_request_attempts == 3:
# We already tried the same request three times, bail out
self.request_cache.pop("chaincrawl", cache.number)
cache.crawl_future.set_result(None)
return
cache.current_request_attempts += 1
await self.send_crawl_request(cache.peer,
cache.peer.public_key.key_to_bin(),
start, stop)
await self.send_next_partial_chain_crawl_request(cache)
async def send_next_partial_chain_crawl_request(self, cache):
"""
Send the next partial crawl request, if we are not done yet.
:param cache: The cache that stores progress regarding the chain crawl.
"""
lowest_unknown = self.persistence.get_lowest_sequence_number_unknown(
cache.peer.public_key.key_to_bin())
if cache.known_chain_length and cache.known_chain_length == lowest_unknown - 1:
# At this point, we have all the blocks we need
self.request_cache.pop("chaincrawl", cache.number)
cache.crawl_future.set_result(None)
return
if not cache.known_chain_length:
# Do we know the chain length of the crawled peer? If not, make sure we get to know this first.
blocks = await self.send_crawl_request(
cache.peer, cache.peer.public_key.key_to_bin(), -1, -1)
if not blocks:
self.request_cache.pop("chaincrawl", cache.number)
cache.crawl_future.set_result(None)
return
cache.known_chain_length = blocks[0].sequence_number
await self.send_next_partial_chain_crawl_request(cache)
return
latest_block = self.persistence.get_latest(
cache.peer.public_key.key_to_bin())
if not latest_block:
# We have no knowledge of this peer but we have the length of the chain.
# Simply send a request from the genesis block to the known chain length.
self.perform_partial_chain_crawl(cache, 1,
cache.known_chain_length)
return
elif latest_block and lowest_unknown == latest_block.sequence_number + 1:
# It seems that we filled all gaps in the database; check whether we can do one final request
if latest_block.sequence_number < cache.known_chain_length:
self.perform_partial_chain_crawl(
cache, latest_block.sequence_number + 1,
cache.known_chain_length)
else:
self.request_cache.pop("chaincrawl", cache.number)
cache.crawl_future.set_result(None)
return
start, stop = self.persistence.get_lowest_range_unknown(
cache.peer.public_key.key_to_bin())
self.perform_partial_chain_crawl(cache, start, stop)
@synchronized
@lazy_wrapper(GlobalTimeDistributionPayload, CrawlRequestPayload)
def received_crawl_request(self, peer, dist, payload):
self.logger.info("Received crawl request from node %s for range %d-%d",
hexlify(peer.public_key.key_to_bin())[-8:],
payload.start_seq_num, payload.end_seq_num)
start_seq_num = payload.start_seq_num
end_seq_num = payload.end_seq_num
# It could be that our start_seq_num and end_seq_num are negative. If so, convert them to positive numbers,
# based on the last block of ones chain.
if start_seq_num < 0:
last_block = self.persistence.get_latest(payload.public_key)
start_seq_num = max(GENESIS_SEQ, last_block.sequence_number + start_seq_num + 1) \
if last_block else GENESIS_SEQ
if end_seq_num < 0:
last_block = self.persistence.get_latest(payload.public_key)
end_seq_num = max(GENESIS_SEQ, last_block.sequence_number + end_seq_num + 1) \
if last_block else GENESIS_SEQ
blocks = self.persistence.crawl(payload.public_key,
start_seq_num,
end_seq_num,
limit=self.settings.max_crawl_batch)
total_count = len(blocks)
if total_count == 0:
global_time = self.claim_global_time()
response_payload = EmptyCrawlResponsePayload(payload.crawl_id)
dist = GlobalTimeDistributionPayload(global_time)
packet = self._ez_pack(self._prefix, 7, [dist, response_payload],
False)
self.endpoint.send(peer.address, packet)
else:
self.send_crawl_responses(blocks, peer, payload.crawl_id)
def send_crawl_responses(self, blocks, peer, crawl_id):
"""
Answer a peer with crawl responses.
"""
for ind, block in enumerate(blocks):
self.send_crawl_response(block, crawl_id, ind + 1, len(blocks),
peer)
self.logger.info("Sent %d blocks", len(blocks))
@synchronized
def sanitize_database(self):
"""
DANGER! USING THIS MAY CAUSE DOUBLE SPENDING IN THE NETWORK.
ONLY USE IF YOU KNOW WHAT YOU ARE DOING.
This method removes all of the invalid blocks in our own chain.
"""
self.logger.error("Attempting to recover %s", self.DB_CLASS.__name__)
block = self.persistence.get_latest(
self.my_peer.public_key.key_to_bin())
if not block:
# There is nothing to corrupt, we're at the genesis block.
self.logger.debug(
"No latest block found when trying to recover database!")
return
validation = self.validate_persist_block(block)
while validation[0] != ValidationResult.partial_next and validation[
0] != ValidationResult.valid:
# The latest block is invalid, remove it.
self.persistence.remove_block(block)
self.logger.error("Removed invalid block %d from our chain",
block.sequence_number)
block = self.persistence.get_latest(
self.my_peer.public_key.key_to_bin())
if not block:
# Back to the genesis
break
validation = self.validate_persist_block(block)
self.logger.error("Recovered database, our last block is now %d",
block.sequence_number if block else 0)
def persistence_integrity_check(self):
"""
Perform an integrity check of our own chain. Recover it if needed.
"""
block = self.persistence.get_latest(
self.my_peer.public_key.key_to_bin())
if not block:
return
validation = self.validate_persist_block(block)
if validation[0] != ValidationResult.partial_next and validation[
0] != ValidationResult.valid:
self.logger.error("Our chain did not validate. Result %s",
repr(validation))
self.sanitize_database()
def send_crawl_response(self, block, crawl_id, index, total_count, peer):
self.logger.debug("Sending block for crawl request to %s (%s)", peer,
block)
# Don't answer with any invalid blocks.
validation = self.validate_persist_block(block)
if validation[0] == ValidationResult.invalid and total_count > 0:
# We send an empty block to the crawl requester if no blocks should be sent back
self.logger.error(
"Not sending crawl response, the block is invalid. Result %s",
repr(validation))
self.persistence_integrity_check()
return
global_time = self.claim_global_time()
payload = CrawlResponsePayload.from_crawl(block, crawl_id, index,
total_count)
dist = GlobalTimeDistributionPayload(global_time)
packet = self._ez_pack(self._prefix, 3, [dist, payload], False)
self.endpoint.send(peer.address, packet)
@synchronized
@lazy_wrapper_unsigned_wd(GlobalTimeDistributionPayload,
CrawlResponsePayload)
async def received_crawl_response(self, source_address, dist, payload,
data):
await self.received_half_block(
source_address,
data[:-12]) # We cut off a few bytes to make it a BlockPayload
block = self.get_block_class(payload.type).from_payload(
payload, self.serializer)
cache = self.request_cache.get("crawl", payload.crawl_id)
if cache:
cache.received_block(block, payload.total_count)
@lazy_wrapper_unsigned_wd(GlobalTimeDistributionPayload,
EmptyCrawlResponsePayload)
def received_empty_crawl_response(self, source_address, dist, payload,
data):
cache = self.request_cache.get("crawl", payload.crawl_id)
if cache:
self.logger.info(
"Received empty crawl response for crawl with ID %d",
payload.crawl_id)
cache.received_empty_response()
def get_chain_length(self):
"""
Return the length of your own chain.
"""
latest_block = self.persistence.get_latest(
self.my_peer.public_key.key_to_bin())
return 0 if not latest_block else latest_block.sequence_number
@synchronized
def create_introduction_request(self,
socket_address,
extra_bytes=b'',
new_style=False):
extra_bytes = struct.pack('>l', self.get_chain_length())
return super().create_introduction_request(socket_address, extra_bytes,
new_style)
@synchronized
def create_introduction_response(self,
lan_socket_address,
socket_address,
identifier,
introduction=None,
extra_bytes=b'',
prefix=None,
new_style=False):
extra_bytes = struct.pack('>l', self.get_chain_length())
return super(TrustChainCommunity, self).create_introduction_response(
lan_socket_address, socket_address, identifier, introduction,
extra_bytes, prefix, new_style)
@synchronized
def introduction_response_callback(self, peer, dist, payload):
chain_length = None
if payload.extra_bytes:
chain_length = struct.unpack('>l', payload.extra_bytes)[0]
if peer.address in self.network.blacklist: # Do not crawl addresses in our blacklist (trackers)
return
# Check if we have pending crawl requests for this peer
has_intro_crawl = self.request_cache.has(
"introcrawltimeout", IntroCrawlTimeout.get_number_for(peer))
has_chain_crawl = self.request_cache.has(
"chaincrawl", ChainCrawlCache.get_number_for(peer))
if has_intro_crawl or has_chain_crawl:
self.logger.debug(
"Skipping crawl of peer %s, another crawl is pending", peer)
return
if self.settings.crawler:
self.crawl_chain(peer, latest_block_num=chain_length)
else:
known_blocks = self.persistence.get_number_of_known_blocks(
public_key=peer.public_key.key_to_bin())
if known_blocks < 1000 or random.random() > 0.5:
self.request_cache.add(IntroCrawlTimeout(self, peer))
self.crawl_lowest_unknown(peer, latest_block_num=chain_length)
async def unload(self):
self.logger.debug("Unloading the TrustChain Community.")
self.shutting_down = True
await self.request_cache.shutdown()
await super(TrustChainCommunity, self).unload()
# Close the persistence layer
self.persistence.close()
class RemoteQueryCommunity(Community):
"""
Community for general purpose SELECT-like queries into remote Channels database
"""
master_peer = Peer(
unhexlify(
"4c69624e61434c504b3a667b8dee4645475512c0780990cfaca234ad19c5dabcb065751776"
"b75a4b4210c06e2eb4d8bbf4a775ed735eb16bbc3e44193479ad7426d7cd1067807f95b696"
)
)
def __init__(self, my_peer, endpoint, network, metadata_store, settings=None, notifier=None):
super(RemoteQueryCommunity, self).__init__(my_peer, endpoint, network)
self.notifier = notifier
self.max_peers = 60
self.settings = settings or RemoteQueryCommunitySettings()
self.mds = metadata_store
# This set contains all the peers that we queried for subscribed channels over time.
# It is emptied regularly. The purpose of this set is to work as a filter so we never query the same
# peer twice. If we do, this should happen realy rarely
# TODO: use Bloom filter here instead. We actually *want* it to be all-false-positives eventually.
self.queried_subscribed_channels_peers = set()
self.queried_peers_limit = 1000
self.add_message_handler(RemoteSelectPayload, self.on_remote_select)
self.add_message_handler(SelectResponsePayload, self.on_remote_select_response)
self.request_cache = RequestCache()
def get_random_peers(self, sample_size=None):
# Randomly sample sample_size peers from the complete list of our peers
all_peers = self.get_peers()
if sample_size is not None and sample_size < len(all_peers):
return sample(all_peers, sample_size)
return all_peers
def send_remote_select(self, peer, **kwargs):
request = SelectRequest(self.request_cache, hexlify(peer.mid), kwargs)
self.request_cache.add(request)
self.ez_send(peer, RemoteSelectPayload(request.number, json.dumps(kwargs).encode('utf8')))
def send_remote_select_to_many(self, **kwargs):
for p in self.get_random_peers(self.settings.max_query_peers):
self.send_remote_select(p, **kwargs)
def send_remote_select_subscribed_channels(self, peer):
request_dict = {
"metadata_type": [CHANNEL_TORRENT],
"subscribed": True,
"attribute_ranges": (("num_entries", 1, None),),
}
self.send_remote_select(peer, **request_dict)
@lazy_wrapper(RemoteSelectPayload)
async def on_remote_select(self, peer, request):
request_sanitized = sanitize_query(json.loads(request.json), self.settings.max_response_size)
db_results = await self.mds.MetadataNode.get_entries_threaded(**request_sanitized)
if not db_results:
return
index = 0
while index < len(db_results):
data, index = entries_to_chunk(db_results, self.settings.maximum_payload_size, start_index=index)
self.ez_send(peer, SelectResponsePayload(request.id, data))
@lazy_wrapper(SelectResponsePayload)
async def on_remote_select_response(self, peer, response):
request = self.request_cache.get(hexlify(peer.mid), response.id)
if request is None:
return
# Check for limit on the number of packets per request
if request.packets_limit > 1:
request.packets_limit -= 1
else:
self.request_cache.pop(hexlify(peer.mid), response.id)
# We use responses for requests about subscribed channels to bump our local channels ratings
peer_vote = peer if request.request_kwargs.get("subscribed", None) is True else None
result = await self.mds.process_compressed_mdblob_threaded(response.raw_blob, peer_vote_for_channels=peer_vote)
# Maybe move this callback to MetadataStore side?
if self.notifier:
new_channels = [
md.to_simple_dict()
for md, result in result
if md and md.metadata_type == CHANNEL_TORRENT and result == UNKNOWN_CHANNEL and md.origin_id == 0
]
if new_channels:
self.notifier.notify(
NTFY.CHANNEL_DISCOVERED, {"results": new_channels, "uuid": str(CHANNELS_VIEW_UUID)}
)
def introduction_response_callback(self, peer, dist, payload):
if peer.address in self.network.blacklist or peer.mid in self.queried_subscribed_channels_peers:
return
if len(self.queried_subscribed_channels_peers) >= self.queried_peers_limit:
self.queried_subscribed_channels_peers.clear()
self.queried_subscribed_channels_peers.add(peer.mid)
self.send_remote_select_subscribed_channels(peer)
async def unload(self):
await self.request_cache.shutdown()
await super(RemoteQueryCommunity, self).unload()
class MyCommunity(Community):
community_id = os.urandom(20)
def __init__(self, *args, **kwargs):
super(MyCommunity, self).__init__(*args, **kwargs)
self.request_cache = RequestCache()
def unload(self):
self.request_cache.shutdown()
super(MyCommunity, self).unload()
def finish_ping(self, cache, include=True):
global RESULTS
print(cache.hostname, cache.address, time.time() - cache.starttime) # noqa: T001
if include:
if (cache.hostname, cache.address) in RESULTS:
RESULTS[(cache.hostname, cache.address)].append(time.time() - cache.starttime)
else:
RESULTS[(cache.hostname, cache.address)] = [time.time() - cache.starttime]
elif (cache.hostname, cache.address) not in RESULTS:
RESULTS[(cache.hostname, cache.address)] = []
self.next_ping()
def next_ping(self):
global CHECK_QUEUE
if CHECK_QUEUE:
hostname, address = CHECK_QUEUE.pop()
packet = self.create_introduction_request(UDPv4Address(*address))
self.request_cache.add(PingCache(self, hostname, address, time.time()))
self.endpoint.send(address, packet)
else:
get_event_loop().stop()
def introduction_response_callback(self, peer, dist, payload):
if self.request_cache.has(u"introping", payload.identifier):
cache = self.request_cache.pop(u"introping", payload.identifier)
self.finish_ping(cache)
def started(self):
global CHECK_QUEUE
dnsmap = {}
for (address, port) in DISPERSY_BOOTSTRAPPER['init']['dns_addresses']:
try:
ip = gethostbyname(address)
dnsmap[(ip, port)] = address
except OSError:
pass
unknown_name = '*'
for (ip, port) in DISPERSY_BOOTSTRAPPER['init']['ip_addresses']:
hostname = dnsmap.get((ip, port), None)
if not hostname:
hostname = unknown_name
unknown_name = unknown_name + '*'
CHECK_QUEUE.append((hostname, (ip, port)))
CHECK_QUEUE = CHECK_QUEUE * CONST_REQUESTS
self.next_ping()
class EndpointServer(Community):
"""
Make some small modifications to the Community to allow it a dynamic prefix.
We will also only answer introduction requests.
"""
community_id = os.urandom(20)
def __init__(self, endpoint):
my_peer = Peer(default_eccrypto.generate_key(u"very-low"))
self.signature_length = default_eccrypto.get_signature_length(
my_peer.public_key)
super().__init__(my_peer, endpoint, Network())
self.request_cache = RequestCache()
self.endpoint.add_listener(
self
) # Listen to all incoming packets (not just the fake community_id).
self.churn_strategy = TrackerChurn(self)
self.churn_task = self.register_task("churn",
self.churn_strategy.take_step,
interval=10)
def on_packet(self, packet, warn_unknown=False):
source_address, data = packet
try:
probable_peer = self.network.get_verified_by_address(
source_address)
if probable_peer:
probable_peer.last_response = time.time()
if data[22] == 246:
self.on_generic_introduction_request(source_address, data,
data[:22])
if data[22] == 245:
self.on_generic_introduction_response(source_address, data,
data[:22])
elif warn_unknown:
self.logger.warning("Tracker received unknown message %s",
str(data[22]))
except Exception:
traceback.print_exc()
def on_generic_introduction_request(self, source_address, data, prefix):
auth, dist, payload = self._ez_unpack_auth(IntroductionRequestPayload,
data)
peer = Peer(auth.public_key_bin, source_address)
peer.address = UDPv4LANAddress(*payload.source_lan_address)
peer.last_response = time.time()
service_id = prefix[2:]
self.on_peer_introduction_request(peer, source_address, service_id)
self.network.add_verified_peer(peer)
self.network.discover_services(peer, [
service_id,
])
intro_peers = [
p for p in self.network.get_peers_for_service(service_id)
if not p == peer
]
if intro_peers:
intro_peer = random.choice(intro_peers)
else:
intro_peer = None
packet = self.create_introduction_response(payload.destination_address,
peer.address,
payload.identifier,
introduction=intro_peer,
prefix=prefix)
self.endpoint.send(peer.address, packet)
def send_ping(self, peer):
service = random.choice(tuple(
self.network.get_services_for_peer(peer)))
prefix = b'\x00' + self.version + service
packet = self.create_introduction_request(peer.address, prefix=prefix)
cache = TrackerPing(self.request_cache, self.global_time, self.network,
peer, service)
self.request_cache.add(cache)
self.endpoint.send(peer.address, packet)
def on_generic_introduction_response(self, source_address, data, prefix):
auth, dist, payload = self._ez_unpack_auth(IntroductionResponsePayload,
data)
if not self.request_cache.has('tracker-ping', payload.identifier):
return
self.request_cache.pop('tracker-ping', payload.identifier)
if payload.peer_limit_reached:
peer = Peer(auth.public_key_bin, source_address)
services = self.network.get_services_for_peer(peer)
services.discard(prefix[2:])
if not services:
self.network.remove_peer(peer)
def on_peer_introduction_request(self, peer, source_address, service_id):
"""
A hook to collect anonymized statistics about total peer count
"""
def get_peer_for_introduction(self, exclude=None, new_style=False):
"""
We explicitly provide create_introduction_response with a peer.
If on_generic_introduction_request provides None, this method should not suggest a peer.
More so as the get_peer_for_introduction peer would be for the DiscoveryCommunity.
"""
return None
async def unload(self):
await self.request_cache.shutdown()
await super().unload()