def __send_command(self, command_type: bytes, data: bytes, expect_reply: bool = False, blacklist: Set[InstanceReference] = set()) -> Subject:
# If we expect replies to this command, create a subject
reply_subject = None
if(expect_reply):
reply_subject = Subject()
count = 0
# Loop over each repeater
for repeater in self.__repeaters:
# Is this a blacklisted repeater?
if(repeater in blacklist):
# Yes, skip
continue
# Send command
subject = self.__send_command_to(command_type, data, repeater, expect_reply)
count += 1
# Do we expect a reply?
if(expect_reply):
# Yes, connect to subject
subject.subscribe(reply_subject.on_next)
Log.debug("Broadcasted a command to {} repeater/s".format(count))
# Return reply subject
return reply_subject
def on_query_answer(answer: InstanceInformation):
Log.debug("Found AIP peer in group '{}'".format(
group.decode("utf-8")))
# Create a subject so we know when this peer has been greeted
self.__on_peer_greet[
answer.instance_reference] = rx.subject.Subject()
# Add to group
self.__query_groups[group].add_peer(answer.instance_reference)
# Are we already connected to this peer?
if (answer.instance_reference in self.__reachable_peers):
# No need to greet, already connected
self.__new_group_peer[group].on_next(
answer.instance_reference)
return
# When is has been greeted, notify the group subject
self.__on_peer_greet[answer.instance_reference].subscribe(
self.__new_group_peer[group].on_next)
# Inquire
self.__muxer.inquire(self.__instance,
answer.instance_reference,
answer.connection_methods)
def inquire_via_paths(self, instance: Instance,
destination: InstanceReference,
strategies: List[PathStrategy]):
# Create an inquiry
inquiry = Inquiry(destination)
self.__inquiries[inquiry.id] = inquiry
packets = 0
# Loop over each path to try
for strategy in strategies:
# Do we have the network associated with the peer info?
if (strategy.first_hop.NETWORK_TYPE not in self.__networks):
# We don't have this repeater's network
continue
# Loop over the networks that match the type
for network in self.__networks[strategy.first_hop.NETWORK_TYPE]:
# Create a frame containing an inquire packet
frame = Frame(
destination, instance.reference, strategy.path,
BytesIO(MX2.PACKET_INQUIRE + inquiry.id +
instance.application_namespace.encode("utf-8")))
# Send using the network and peer info
Thread(name="MX2 Inquiry",
target=self.__tolerant_inquire,
args=(network, frame, strategy.first_hop,
instance)).start()
packets += 1
Log.debug(
"Sent inquiry via path, resulting in {} packet/s".format(packets))
return inquiry.complete
def __discover_dns_seeds(self):
# Loop over each DNS seed
for seed in LIBPEER_DNS_SEEDS:
# Try and query
try:
# Query for TXT records
answers = resolver.query(seed, "TXT")
# Loop over answers
for answer in answers:
# Loop over strings
for string in answer.strings:
# Split on '/' delimiter
data = string.split(b'/')
# Is it a LibPeer2 entry?
if(not data[0].startswith(b"P2")):
# No, skip
continue
if(data[0] == b"P2M"):
# Seed message
Log.msg("DNS Seed: {}".format(data[1].decode('utf-8')))
elif(data[0] == b"P2D" or data[0] == b"P2A"):
# Domain pointer or IP Address
self.__handle_dns_discovery(data[1], data[2])
except:
pass
Log.debug("DNS Seed Search Complete")
def __notify(self):
while True:
try:
self.__notification_queue.get()()
except Exception as e:
print(traceback.format_exc())
Log.error(
"Exception executing task in STP notification queue: {}".
format(e))
def __init__(self):
from LibPeer2.Debug import Log
Log.msg("Loading frame repeater...")
from LibPeer2.Repeater.FrameRepeater import FrameRepeater
from LibPeer2.Networks.IPv4 import IPv4
Log.msg("Initialising Network...")
network = IPv4("0.0.0.0", 7777)
network.bring_up()
Log.msg("Initialising Frame Repeater...")
repeater = FrameRepeater()
Log.msg("Attaching Network...")
repeater.register_network(network)
Log.info("Frame Repeater is Ready.")
def send_reply(isDefered=True):
# Create some instance information
instance = InstanceInformation(self.__instance.reference,
self.__peer_info)
# Send the instance information in the answer
answer = Answer(instance.serialise(),
query.return_path.copy(), query.identifier)
# Send the answer
self.__send_answer(answer)
if (isDefered):
Log.debug("Sent defered group reply")
def __listen(self):
while True:
try:
# Receive the next datagram
datagram, sender = self.__socket.recvfrom(65536)
# Put the datagram into a buffer
buffer = BytesIO(datagram)
# Create peer info
info = IPv4PeerInfo(sender[0], sender[1])
# Read the datagram type
dgram_type = buffer.read(1)
# Regular data packet
if(dgram_type == DGRAM_DATA):
# Create a new receiption
receiption = Receiption(buffer, info, self)
# Pass up
self.incoming_receiption.on_next(receiption)
elif(dgram_type == DGRAM_INQUIRE):
# Respond with instance information
for instance in self.__advertised_instances:
# Send the instance information as a single datagram
self.__socket.sendto(DGRAM_INSTANCE + instance.serialise().read(), sender)
elif(dgram_type == DGRAM_INSTANCE):
# Create the instance reference
instance_reference = InstanceReference.deserialise(buffer)
# Is the instance one we advertise?
if(instance_reference in self.__advertised_instances):
# Yes, skip
continue
# Create the advertisement
advertisement = Advertisement(instance_reference, info)
# Send to the application
self.incoming_advertisment.on_next(advertisement)
except Exception as e:
Log.error("Exception on incoming packet: {}".format(e))
def __loop(self):
while (self.up):
receiption = self.queue.get()
time.sleep(self.delay)
ran_num = random.randint(1, 100)
lost = ran_num <= self.loss_probability * 100
if (not lost):
try:
self.incoming_receiption.on_next(receiption)
except Exception as e:
print(traceback.format_exc())
Log.error("Exception on incoming packet: {}".format(e))
else:
Log.info("NetSim dropped a packet ({} <= {})".format(
ran_num, self.loss_probability * 100))
def __new_stream(self, stream: IngressStream):
# New command, what is it?
command_type = stream.read(1)
# Announce
if(command_type == COMMAND_ANNOUNCE):
# Read announcement
announcement = Announcement.deserialise(stream)
# Get peer info and network
info = self.__muxer.get_peer_info(stream.origin)
network = self.__muxer.get_peer_network(stream.origin)
Log.debug("Peer announced itself with {} attached instance/s".format(len(announcement.instances)))
# Update records
for instance in announcement.instances:
self.instance_info[instance] = info
self.instance_network[instance] = network
# Also add info for the RPP peer
self.instance_info[stream.origin] = info
self.instance_network[stream.origin] = network
# Join
elif(command_type == COMMAND_JOIN):
Log.debug("Repeater peer joined")
# Add as repeater peer
self.__repeaters.add(stream.origin)
# Save instance info for flag lookups
self.instance_info[stream.origin] = self.__muxer.get_peer_info(stream.origin)
self.instance_network[stream.origin] = self.__muxer.get_peer_network(stream.origin)
# Find path
elif(command_type == COMMAND_FIND_PATH):
# Read the query
query = PathQuery.deserialise(stream)
# Is it an instance directly connected to us?
if(query.target in self.instance_info):
# Yes, get the flags
flags = self.__get_instance_flags(stream.origin, query.target)
Log.debug("Servicing query for path to an instance that is currently connected")
self.__query_respond(stream.origin, stream.id, PathResponse([]))
elif(query.ttl > 0):
Log.debug("Forwarding query for path to instance")
# Instance is not directly connected, forward query if still alive
self.__forward_query(stream.origin, stream.id, query)
def __close_session(self, reason):
# Ignore if stream already closed
if (not self.open):
return
Log.debug("Stream closed: {}".format(reason))
self.open = False
# Get all trackers tracking transmissions that have not completed
current_trackers = set(self.segment_trackers[x]
for x in self.in_flight.keys())
# Fail them all
for tracker in current_trackers:
tracker.fail(
IOError(
"The stream was closed before all data was sent. Reason: {}"
.format(reason)))
self.closed.on_completed()
def handle_stream(es: EgressStream):
if (request_type == REQUEST_CAPABILITIES):
Log.debug("Received capabilities request")
capabilities = struct.pack("!B", len(self.__capabilities))
capabilities += b"".join(self.__capabilities)
es.write(capabilities)
es.close()
elif (request_type == REQUEST_ADDRESS):
Log.debug("Received address request")
address = self.__muxer.get_peer_info(es.target)
es.write(address.serialise())
es.close()
elif (request_type == REQUEST_PEERS):
Log.debug("Received peers request")
# Select up to 5 peers to reply with
peers = [
] #[x for x in random.sample(self.__default_group.instances, min(5, len(self.__reachable_peers))) if x in self.__peer_connection_methods]
# Send the count
es.write(struct.pack("!B", len(peers)))
for peer in peers:
# Get the peer's connection methods
methods = self.__peer_connection_methods[peer]
# Create an instance information object
info = InstanceInformation(peer, methods)
# Write the object to the stream
es.write(info.serialise())
def send_group_query():
# Construct a query asking for peers in the group
query = Query(QUERY_GROUP + group)
Log.debug("Joining group '{}'".format(group.decode("utf-8")))
# Create handler for query answers
def on_query_answer(answer: InstanceInformation):
Log.debug("Found AIP peer in group '{}'".format(
group.decode("utf-8")))
# Create a subject so we know when this peer has been greeted
self.__on_peer_greet[
answer.instance_reference] = rx.subject.Subject()
# Add to group
self.__query_groups[group].add_peer(answer.instance_reference)
# Are we already connected to this peer?
if (answer.instance_reference in self.__reachable_peers):
# No need to greet, already connected
self.__new_group_peer[group].on_next(
answer.instance_reference)
return
# When is has been greeted, notify the group subject
self.__on_peer_greet[answer.instance_reference].subscribe(
self.__new_group_peer[group].on_next)
# Inquire
self.__muxer.inquire(self.__instance,
answer.instance_reference,
answer.connection_methods)
# Subscribe to the answer
query.answer.subscribe(on_query_answer)
# Send the query
self.__initiate_query(query, self.__default_group)
def inquire(self, instance: Instance, destination: InstanceReference,
peers: List[PeerInfo]):
# Create an inquiry
inquiry = Inquiry(destination)
self.__inquiries[inquiry.id] = inquiry
if (len(peers) == 0):
raise Exception("why")
packets = 0
# Loop over each peer to try
for peer in peers:
# Do we have the network associated with the peer info?
if (peer.NETWORK_TYPE not in self.__networks):
Log.debug(
"Connection method skipped as there are no networks of type '{}' registered"
.format(peer.NETWORK_TYPE.decode("utf-8")))
# We don't have this peer's network
continue
# Loop over the networks that match the type
for network in self.__networks[peer.NETWORK_TYPE]:
# Create a frame containing an inquire packet
frame = Frame(
destination, instance.reference, PathInfo.empty(),
BytesIO(MX2.PACKET_INQUIRE + inquiry.id +
instance.application_namespace.encode("utf-8")))
# Send using the network and peer info
Thread(name="MX2 Inquiry",
target=self.__tolerant_inquire,
args=(network, frame, peer, instance)).start()
packets += 1
Log.debug("Sent inquiry, resulting in {} packet/s".format(packets))
return inquiry.complete
def __handle_receiption(self, receiption: Receiption):
# Read frame within receiption
try:
frame, instance = Frame.deserialise(receiption.stream,
self.__instances)
except Exception as e:
Log.error(str(e))
return
# Read packet type
packet_type = frame.payload.read(1)
# Determine what to do
if (packet_type == MX2.PACKET_INQUIRE):
Log.debug("Received Inquire Packet")
# First 16 bytes of packet is inquiry id
inquiry_id = frame.payload.read(16)
# Rest of packet indicates desired application name
application_namespace = frame.payload.read().decode("utf-8")
# Does the application namespace match the instance's?
if (instance.application_namespace == application_namespace):
# Yes! Save this instance's information locally for use later
self.__remote_instance_mapping[frame.origin] = (
receiption.network, receiption.peer_info,
frame.via.return_path())
# Reply with a greeting and the inquiry id
self.__send_packet(instance, frame.origin,
BytesIO(MX2.PACKET_GREET + inquiry_id))
elif (packet_type == MX2.PACKET_GREET):
Log.debug("Received Greet Packet")
# We received a greeting!
# Have we received one from this instance before?
if (frame.origin not in self.__remote_instance_mapping):
# No, this is the first (therefore, least latent) method of talking to this instance
self.__remote_instance_mapping[frame.origin] = (
receiption.network, receiption.peer_info,
frame.via.return_path())
# Read inquiry id
inquiry_id = frame.payload.read(16)
# Get ping
ping = 120.0
if (inquiry_id in self.__inquiries):
ping = self.__inquiries[inquiry_id].response_received()
# Save ping
self.__pings[frame.origin] = ping
# Does the instance know that this is now a reachable peer?
if (frame.origin not in instance.reachable_peers):
# No, notify it
instance.reachable_peers.add(frame.origin)
instance.incoming_greeting.on_next(frame.origin)
elif (packet_type == MX2.PACKET_PAYLOAD):
# This is a payload for the next layer to handle, pass it up.
instance.incoming_payload.on_next(Packet(frame))
def process_segment(self, segment):
# We have received a segment from the muxer
# Figure out the segment type
if (isinstance(segment, Acknowledgement)):
# Is this segment still in flight?
if (segment.sequence_number not in self.in_flight):
# We must have resent redundantly
self.redundant_resends += 1
Log.debug("Redundant resend of segment")
return
# We have an acknowledgement segment, remove payload segment from in-flight
del self.in_flight[segment.sequence_number]
# Do we have a tracking object for this?
if (segment.sequence_number in self.segment_trackers):
# Yes, notify it (TODO maybe put this into a queue so as not to block the network rx thread)
self.segment_trackers[segment.sequence_number].complete(
segment.sequence_number)
# What was the time difference?
round_trip = time.time() - segment.timing
# Are we currently at metric window size?
if (self.window_size == METRIC_WINDOW_SIZE):
# Add round trip time to the set
self.segment_trips.add(round_trip)
# Do we have a sample?
if (len(self.segment_trips) >= METRIC_WINDOW_SIZE):
# Update the ping based on the average of the metric segments
self.best_ping = sum(self.segment_trips) / float(
len(self.segment_trips))
# Update the window size now we have our baseline
self.__adjust_window_size(round_trip)
else:
# No, adjust the window size
self.__adjust_window_size(round_trip)
elif (isinstance(segment, Payload)):
# We have a payload segment, run it through the enabled features
for i in range(len(self.features), 0, -1):
segment.data = self.features[i - 1].unwrap(segment.data)
# Is this the next expected segment?
if (self.next_expected_sequence_number == segment.sequence_number):
# Is there anything on the reconstruction dictionary?
if (len(self.reconstruction) > 0):
# Add to reconstruction dict
self.reconstruction[segment.sequence_number] = segment
# Reconstruct and send to application
self.incoming_app_data.on_next(
self.__complete_reconstruction())
else:
# Increment next expected sequence number
self.next_expected_sequence_number += 1
# Just send to the app
self.incoming_app_data.on_next(BytesIO(segment.data))
elif (self.next_expected_sequence_number <
segment.sequence_number):
# We obviously missed a segment, get this one ready for reconstruction
self.reconstruction[segment.sequence_number] = segment
# Acknowledge the segment
acknowledgement = Acknowledgement(segment.sequence_number,
segment.timing)
self.outgoing_segment_queue.put(acknowledgement)
elif (isinstance(segment, Control)):
# We have a control segment, what is it telling us?
if (segment.command == Control.CMD_COMPLETE):
# Close the stream
self.__close_session("The remote peer completed the stream")
if (segment.command == Control.CMD_ABORT):
# Close the stream
self.__close_session(
"The stream was aborted by the remote peer")
def __timeout(self):
Log.warn("Inquiry timed out")
self.complete.on_error(TimeoutError("The inquiry timed out."))
def __handle_packet(self, packet: Packet):
# We have a packet from the muxer, deserialise it
message = Message.deserialise(packet.stream)
# What type of message do we have?
if (isinstance(message, RequestSession)):
# Skip if we have already handled this request
if (message.session_id in self.__negotiations):
return
# A peer wants to initiate a session with us
# Create a negotiation object (consider it negotiated as we are about to send back our negotiation)
negotiation = StreamNegotiation(message.session_id,
message.in_reply_to,
message.feature_codes,
StreamNegotiation.STATE_NEGOTIATED,
packet.origin, True)
# Add to negotiations
self.__negotiations[message.session_id] = negotiation
# Figure out which features we can apply
features = Feature.get_features(message.feature_codes)
# Get our feature codes
feature_codes = [x.IDENTIFIER for x in features]
# Construct a reply
reply = NegotiateSession(negotiation.session_id, feature_codes,
message.timing)
# Repeatedly send the negotiation
repeater = Repeater(
10, 12, lambda: self.__send_packet(negotiation.remote_instance,
reply.serialise()),
"STP Stream Negotiation")
# Save the repeater against the negotiation
negotiation.negotiate_repeater = repeater
elif (isinstance(message, NegotiateSession)):
# We are getting a negotiation reply from a peer
# Do we have a negotiation open with this peer?
if (message.session_id not in self.__negotiations):
# TODO send cleanup
return
# Get the negotiation
negotiation: StreamNegotiation = self.__negotiations[
message.session_id]
# Cancel the request repeater
if (negotiation.request_repeater != None):
negotiation.request_repeater.cancel()
# Set the ping value
negotiation.ping = time.time() - message.reply_timing
# Make sure we have all the features they negotiated upon
features = Feature.get_features(message.feature_codes)
if (len(features) != len(message.feature_codes)):
# TODO send cleanup
return
# Update the features list
negotiation.feature_codes = message.feature_codes
# Reply with a begin session message
reply = BeginSession(negotiation.session_id, message.timing)
# Send the reply
self.__send_packet(negotiation.remote_instance, reply.serialise())
# Make sure the negotiation is in the right state
if (negotiation.state != StreamNegotiation.STATE_REQUESTED):
return
# Update the negotiaiton state
negotiation.state = StreamNegotiation.STATE_ACCEPTED
# Setup the session
self.__setup_session(negotiation)
elif (isinstance(message, BeginSession)):
# We are getting a negotiation reply from a peer
# Do we have a negotiation open with this peer?
if (message.session_id not in self.__negotiations):
# TODO send cleanup
return
# Get the negotiation
negotiation: StreamNegotiation = self.__negotiations[
message.session_id]
# Cancel the negotiate repeater
if (negotiation.negotiate_repeater != None):
negotiation.negotiate_repeater.cancel()
# Make sure the negotiation is in the right state
if (negotiation.state != StreamNegotiation.STATE_NEGOTIATED):
# TODO send cleanup
return
# Update the negotiation state
negotiation.state = StreamNegotiation.STATE_ACCEPTED
# Set the ping value
negotiation.ping = time.time() - message.reply_timing
# Setup the session
self.__setup_session(negotiation)
# Cleanup the negotiation
del self.__negotiations[message.session_id]
elif (isinstance(message, SegmentMessage)):
# Do we have a session open?
if (message.session_id not in self.__open_sessions):
Log.debug("Received segment for non-open session")
# Skip
return
# Is there a valid negotiation still open?
if (message.session_id in self.__negotiations):
# Cleanup the negotiation
del self.__negotiations[message.session_id]
# Get the session
session: Session = self.__open_sessions[message.session_id]
# Give the session the segment
session.process_segment(message.segment)
def __handle_query(self, stream: IngressStream):
# Deserialise the query
query = Query.deserialise(stream)
# Have we come across this query before?
if (query.identifier in self.__handled_query_ids):
# Don't forward
return
# Mark as handled
self.__handled_query_ids.add(query.identifier)
# Create a replies counter
self.__query_response_count[query.identifier] = query.max_replies
# Append the originator of the stream to the query reply path
query.return_path.append(stream.origin)
# Increment the query hops
query.hops += 1
# Find query type
query_type = query.data[:1]
if (query_type == QUERY_GROUP):
# Get the group identifier
group = query.data[1:]
Log.debug("Received group query for group '{}'".format(
group.decode("utf-8")))
# Are we not in this group, but joining all?
if (self.__join_all_groups) and (group not in self.__query_groups):
Log.debug("Automatically joining new group")
# Join group
self.__join_query_group(group)
# Are we in this group?
if (group in self.__query_groups):
# Yes create a function for sending the answer
def send_reply(isDefered=True):
# Create some instance information
instance = InstanceInformation(self.__instance.reference,
self.__peer_info)
# Send the instance information in the answer
answer = Answer(instance.serialise(),
query.return_path.copy(), query.identifier)
# Send the answer
self.__send_answer(answer)
if (isDefered):
Log.debug("Sent defered group reply")
# Do we have peer info to send yet?
if (len(self.__peer_info) > 0):
# Yes, do it
Log.debug("Responding to query for group '{}'".format(
group.decode("utf-8")))
send_reply(False)
else:
# No, wait for peer info
self.__new_peer_info.pipe(rx.operators.take(1)).subscribe(
on_completed=send_reply)
Log.debug(
"Responding to query for group '{}' but defering response until we have peer info to send"
.format(group.decode("utf-8")))
# This is a query for a group, forward on to default group
self.__send_query(query, self.__default_group)
elif (query_type == QUERY_APPLICATION):
# Get the application namespace
namespace = query.data[1:]
Log.debug("Received application query for '{}'".format(
namespace.decode("utf-8")))
# Are we in the group for this namespace?
if (namespace in self.__query_groups):
# Yes, find relevent ApplicationInformation
for app in self.__application_information:
# Is this app relevent? TODO: Use a dictionary
if (app.namespace_bytes == namespace):
# Yes, create instance information
instance = InstanceInformation(app.instance,
self.__peer_info)
# Send the instance information in the answer
self.__send_answer(
Answer(instance.serialise(),
query.return_path.copy(), query.identifier))
# Forward on to the group
self.__send_query(query, self.__query_groups[namespace])
elif (query_type == QUERY_APPLICATION_RESOURCE):
# Read the label
label = query.data[1:33]
# Read the application namespace
namespace = query.data[33:]
Log.debug(
"Received application resource query for application '{}'".
format(namespace.decode("utf-8")))
# Are we in the group for this namespace?
if (namespace in self.__query_groups):
# Yes, find relevent ApplicationInformation
for app in self.__application_information:
# Is this app relevent? TODO: Use a dictionary
if (app.namespace_bytes == namespace
and label in app.resources):
# Yes, create instance information
instance = InstanceInformation(app.instance,
self.__peer_info)
Log.debug(
"Responded to peer requesting resource associated with this peer"
)
# Send the instance information in the answer
self.__send_answer(
Answer(instance.serialise(),
query.return_path.copy(), query.identifier))
# Forward on to the group
self.__send_query(query, self.__query_groups[namespace])