class PlotMagnitude: def __init__(self): self.ens_receiver = None self.ens_reader = None # Codec to decode the data from the file self.codec = AdcpCodec(is_udp=False) self.codec.EnsembleEvent += self.process_ensemble_codec self.ens_codec_count = 0 sns.plt.axis([0, 10, 0, 1]) sns.plt.ion() self.fig = sns.plt.figure() self.ax = self.fig.add_subplot(111) # Create an axes. sns.plt.setp(self.ax.xaxis.get_majorticklabels(), rotation=45) # Flag to display the colorbar only once self.cbar_display = True self.mag_df = None def process(self, file_path): """ Read the file and start a thread to monitor the incoming ensembles. :param file_path: File path the read files :return: """ # Process the file self.process_file(file_path) logger.info("Completed File reader") logger.info("Ensemble Codec Count: " + str(self.ens_codec_count)) def process_file(self, file_path): """ Process the file given. This read from the file and add it to the codec. The codec will then decode the data and pass it to the UDP port. """ # Check if the file exist if os.path.exists(file_path): logger.info("Open file: " + file_path) # Open the file f = open(file_path, "rb") # Add the data from the file to the codec data = f.read(4096) while len(data) > 0: # Add data to codec self.codec.add(data) # Read next block from the file data = f.read(4096) # Close the file f.close() # Plot final results sns.heatmap(self.mag_df, cbar=self.cbar_display ) # Set flag to only display colorbar once sns.plt.pause(0.000001) # Pause to refresh the display self.cbar_display = False while True: input("Press enter to continue") break else: logger.error("File does not exist") def process_ensemble_codec(self, sender, ens): """ Receive and process the incoming ensemble directly from the codec. This data was process and passed as an Ensemble object. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ if ens.IsEnsembleData: #print("Codec: " + str(ens.EnsembleData.EnsembleNumber)) self.ens_codec_count += 1 if ens.IsEarthVelocity: #print(len(ens.EarthVelocity.Velocities)) earthVel_np = np.array( ens.EarthVelocity.Velocities ) # Create Numpy array for the velocity data #earthVel_np = np.array(ens.BeamVelocity.Velocities) df = pd.DataFrame( columns=['East', 'North', 'Vertical', 'Error'], data=earthVel_np) # Create a description(name) for the columns print(df[:75]) #print(df.shape) # Add Magnitude and Direction to the dataframe df["mag"] = 0 df["dir"] = 0 #df.index.name = "index" for index, row in df.iterrows(): east = row["East"] north = row["North"] vert = row["Vertical"] mag = 0.0 dir = 0.0 # Calculate the mag and dir if the data is good if not math.isclose(east, Ensemble.Ensemble.BadVelocity, rel_tol=0.001, abs_tol=0.001) and \ not math.isclose(north, Ensemble.Ensemble.BadVelocity, rel_tol=0.001, abs_tol=0.001) and \ not math.isclose(vert, Ensemble.Ensemble.BadVelocity, rel_tol=0.001, abs_tol=0.001): mag = math.sqrt( math.pow(east, 2) + math.pow(north, 2) + math.pow(vert, 2)) dir = math.atan2(north, east) * (180.0 / math.pi) df.loc[index, "mag"] = mag df.loc[index, "dir"] = dir #print(df) # Line #self.ax.cla() # Clear the axis so it will not repopulate the list #df.plot(ax=self.ax, y="mag") # Set y to the column to plot, Pass the axes to plot. #plt.draw() # Draw instead of show to update the plot in ion mode. #plt.pause(0.0001) # Pause the plot so it can refreshed and seen # Heatmap #piv = pd.pivot_table(df, values="mag", index=df.index) # Create pivot table to just get 1 column of data #sns.heatmap(piv, cbar=self.cbar_display) # Set flag to only display colorbar once #sns.plt.pause(0.0001) #self.cbar_display = False # Create a temp dataframe # Then combine it with the master dataframe df_mag = pd.DataFrame( columns=[str(ens.EnsembleData.EnsembleNumber)], index=df.index) df_mag[str(ens.EnsembleData.EnsembleNumber)] = df['mag'] #print(df_mag) if self.mag_df is None: self.mag_df = df_mag # Create the initial dataframe else: self.mag_df = pd.concat( [self.mag_df, df_mag], axis=1) # Combine the dataframes into one large one
class waypoint_creator: def __init__(self): # Codec to decode the data from the file self.codec = AdcpCodec(55057) self.codec.EnsembleEvent += self.process_ensemble_codec self.map_osm = None self.ens_codec_count = 0 def process(self, file_path): """ Read the file and start a thread to monitor the incoming ensembles. :param file_path: File path the read files :return: """ # Process the file self.process_file(file_path) self.map_osm.save('osm.html') logger.info("Completed File reader") logger.info("Ensemble Codec Count: " + str(self.ens_codec_count)) def process_file(self, file_path): """ Process the file given. This read from the file and add it to the codec. The codec will then decode the data and pass it to the UDP port. """ # Check if the file exist if os.path.exists(file_path): logger.info("Open file: " + file_path) # Open the file f = open(file_path, "rb") # Add the data from the file to the codec data = f.read(4096) while len(data) > 0: # Add data to codec self.codec.add(data) # Read next block from the file data = f.read(4096) # Close the file f.close() else: logger.error("File does not exist") def process_ensemble_codec(self, sender, ens): """ Receive and process the incoming ensemble directly from the codec. This data was process and passed as an Ensemble object. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ if ens.IsEnsembleData: #print("Codec: " + str(ens.EnsembleData.EnsembleNumber)) self.ens_codec_count += 1 if ens.IsNmeaData: if ens.NmeaData.GPGGA is not None: if self.map_osm is None: self.map_osm = folium.Map(location=[ ens.NmeaData.GPGGA.latitude, ens.NmeaData.GPGGA.longitude ], zoom_start=16) marker_msg = "Lat: " + str( ens.NmeaData.GPGGA.latitude) + " Lon:" + str( ens.NmeaData.GPGGA.longitude) folium.Marker([ ens.NmeaData.GPGGA.latitude, ens.NmeaData.GPGGA.longitude ], popup=marker_msg).add_to(self.map_osm)
class EmitAdcpFile: """ Open a file contains ensembles from a waves burst. Process the data into a waves burst file. Send the data to the UDP port. Send the data to the RabbitMQ. Get the data directly from the codec. """ def __init__(self, ens_in_burst, path, url="localhost", user="******", pw="guest"): """ Initialize the processor. Get the number ensembles per burst and process the data and store the recorded MATLAB file to the path given. :param ens_in_burst: Number of ensembles per waves burst. :param path: File path to record the MATLAB file. :param url: URL to RabbitMQ server. :param user: Username. :param pw: Password. """ self.ens_receiver = None self.ens_reader = None # Codec to decode the data from the file self.codec = AdcpCodec(55057) self.codec.EnsembleEvent += self.process_ensemble_codec self.codec.enable_waveforce_codec(ens_in_burst, path, 32.123, 117.234, 1, 2, 3, 12.456) # Enable WaveForce codec self.ens_count = 0 self.ens_codec_count = 0 self.prev_ens_num = 0 self.missing_ens = 0 self.rabbit = rabbitmq_topic() self.rabbit.connect("ADCP", url, user, pw) def process(self, file_path): """ Read the file and start a thread to monitor the incoming ensembles. :param file_path: File path the read files """ # Create ensemble receiver self.ens_receiver = EnsembleReceiver() self.ens_receiver.EnsembleEvent += self.process_ensemble # Start thread to monitor incoming ensembles # Connect to ensemble server self.ens_reader = threading.Thread(name='EnsFileReader', target=self.ens_receiver.connect, args=[55057]).start() # Process the file self.process_file(file_path) # Stop the receiver self.ens_receiver.close() logger.info("Completed File reader") logger.info("Ensemble UDP Count: " + str(self.ens_count)) if self.missing_ens > 0: logger.info("Missing Ensembles from UDP: " + str(self.missing_ens)) logger.info("Ensemble Codec Count: " + str(self.ens_codec_count)) def process_file(self, file_path): """ Process the file given. This read from the file and add it to the codec. The codec will then decode the data and pass it to the UDP port. """ # Check if the file exist if os.path.exists(file_path): logger.info("Open file: " + file_path) # Open the file f = open(file_path, "rb") # Add the data from the file to the codec data = f.read(4096) while len(data) > 0: # Add data to codec self.codec.add(data) # Read next block from the file data = f.read(4096) # Close the file f.close() else: logger.error("File does not exist") def process_ensemble(self, sender, ens): """ Receive and process the incoming ensemble from the UDP port. This data has been processed through the codec then passed over the UDP port as JSON data. The JSON datasets were then collected and assembled as a JSON ensemble. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ logger.debug("UDP: " + str(ens.EnsembleNumber)) self.ens_count += 1 # Check for missing ensembles if self.prev_ens_num > 0 and self.prev_ens_num + 1 != ens.EnsembleNumber: for msens in range((ens.EnsembleNumber - 1) - self.prev_ens_num): logger.info("Missing Ens: " + str(self.prev_ens_num + msens + 1) + " prev: " + str(self.prev_ens_num) + " cur: " + str(ens.EnsembleNumber)) # add 1 to msens because 0 based self.missing_ens += 1 self.prev_ens_num = ens.EnsembleNumber def process_ensemble_codec(self, sender, ens): """ Receive and process the incoming ensemble directly from the codec. This data was process and passed as an Ensemble object. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ if ens.IsEnsembleData: logger.debug("Codec: " + str(ens.EnsembleData.EnsembleNumber)) self.ens_codec_count += 1 # Publish to RabbitMQ self.emit_ens(ens) def emit_ens(self, ens): """ Emit the ensemble data to the RabbitMQ. :param ens: Ensemble data. """ serial = "0000" if ens.IsEnsembleData: serial = ens.EnsembleData.SerialNumber self.rabbit.send("adcp." + serial + ".data.pb", pickle.dumps(ens))
class PlotMagnitude: def __init__(self): self.ens_receiver = None self.ens_reader = None # Codec to decode the data from the file self.codec = AdcpCodec(is_udp=False) self.codec.EnsembleEvent += self.process_ensemble_codec self.ens_codec_count = 0 # Create a dataframe to hold a rowe for every bin self.df_mag_ens = pd.DataFrame(data=None, columns=['bin', 'ens', 'mag', 'dir']) def process(self, file_path): """ Read the file and start a thread to monitor the incoming ensembles. :param file_path: File path the read files :return: """ # Process the file self.process_file(file_path) logger.info("Completed File reader") logger.info("Ensemble Codec Count: " + str(self.ens_codec_count)) def process_file(self, file_path): """ Process the file given. This read from the file and add it to the codec. The codec will then decode the data and pass it to the UDP port. """ # Check if the file exist if os.path.exists(file_path): logger.info("Open file: " + file_path) # Open the file f = open(file_path, "rb") # Add the data from the file to the codec data = f.read(4096) while len(data) > 0: # Add data to codec self.codec.add(data) # Read next block from the file data = f.read(4096) # Close the file f.close() # Display the Heatmap and save it to HTML file output_file('test.html') hover = HoverTool(tooltips=[ ("index", "$index"), ("(x,y)", "($x, $y)"), ("desc", "@desc"), ]) TOOLS = "hover,save,pan,box_zoom,reset,wheel_zoom" hm = HeatMap( self.df_mag_ens, x='ens', y='bin', values='mag', stat=None, sort_dim={'x': False}, width=1000, spacing_ratio=0.9, tools=TOOLS, toolbar_location="above", # Move toolbar to top toolbar_sticky=False) # Make toolbar not to close to plot # Set min and max axis and invert axis xmin = self.df_mag_ens['ens'].min() xmax = self.df_mag_ens['ens'].max() ymax = self.df_mag_ens['bin'].min() # Swap Min and Max for y axis ymin = self.df_mag_ens['bin'].max() # Swap Min and Max for y axis hm.x_range = Range1d( xmin, xmax) # Set the min and max, so no gaps on edges hm.y_range = Range1d( ymin, ymax) # Set the min and max, so no gaps on edges show(hm) while True: input("Press enter to continue") break else: logger.error("File does not exist") def process_ensemble_codec(self, sender, ens): """ Receive and process the incoming ensemble directly from the codec. This data was process and passed as an Ensemble object. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ if ens.IsEnsembleData: #print("Codec: " + str(ens.EnsembleData.EnsembleNumber)) self.ens_codec_count += 1 if ens.IsEarthVelocity: #print(len(ens.EarthVelocity.Velocities)) earthVel_np = np.array( ens.EarthVelocity.Velocities ) # Create Numpy array for the velocity data #earthVel_np = np.array(ens.BeamVelocity.Velocities) df = pd.DataFrame( columns=['east', 'north', 'vertical', 'error'], data=earthVel_np) # Create a description(name) for the columns for index, row in df.iterrows(): east = row["east"] north = row["north"] vert = row["vertical"] # Calculate the mag and dir if the data is good if not math.isclose(east, Ensemble.Ensemble.BadVelocity, rel_tol=0.001, abs_tol=0.001) and \ not math.isclose(north, Ensemble.Ensemble.BadVelocity, rel_tol=0.001, abs_tol=0.001) and \ not math.isclose(vert, Ensemble.Ensemble.BadVelocity, rel_tol=0.001, abs_tol=0.001): mag = math.sqrt( math.pow(east, 2) + math.pow(north, 2) + math.pow(vert, 2)) dir = math.atan2(north, east) * (180.0 / math.pi) # Add a row for every bin self.df_mag_ens.loc[len(self.df_mag_ens)] = [ index, ens.EnsembleData.EnsembleNumber, mag, dir ]
class SerialEnsembleEmitter: def __init__(self): self.serial_server = None self.serial_server_thread = None self.rabbit = None self.raw_serial_socket = None self.is_alive = True self.codec = None def connect(self, rabbit_url, rabbit_user, rabbit_pw, comm_port, baud, tcp_port="55056"): # Create a RabbitMQ connection self.rabbit = rabbitmq_topic() self.rabbit.connect(exchange="ADCP", host=rabbit_url, user=rabbit_user, pw=rabbit_pw) # Create an ADCP codec self.codec = AdcpCodec() self.codec.EnsembleEvent += self.process_ensemble # Create an ADCP Serial port connection self.serial_server = AdcpSerialPortServer(tcp_port, comm_port, baud) # Start a tcp connection to monitor incoming data and record self.serial_server_thread = threading.Thread(name='AdcpWriter', target=self.create_raw_serial_socket(tcp_port)) self.serial_server_thread.start() def create_raw_serial_socket(self, port): """ Connect to the ADCP serial server. This TCP server outputs data from the serial port. Start reading the data. """ try: # Create socket self.raw_serial_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.raw_serial_socket.connect(('localhost', int(port))) self.raw_serial_socket.settimeout(1) # Set timeout to stop thread if terminated except ConnectionRefusedError as err: logger.error("Serial Send Socket: ", err) except Exception as err: logger.error('Serial Send Socket: ", Error Opening socket', err) # Start to read the raw data self.read_tcp_socket() def read_tcp_socket(self): """ Read the data from the TCP port. This is the raw data from the serial port. Send it to the codec to be decoded. """ while self.is_alive: try: # Read data from socket data = self.raw_serial_socket.recv(4096) # If data exist process if len(data) > 0: self.codec.add(data) except socket.timeout: # Just a socket timeout, continue on pass except Exception as e: logger.error("Exception in reading data.", e) #self.stop_adcp_server() print("Read Thread turned off") def process_ensemble(self, sender, ens): """ Receive an ensemble from the codec. Then pass it to the emitter to pass it to the RabbitMQ server. :param sender: Sender of the ensemble. :param ens: Ensemble data. :return: """ self.emit_ens(ens) def emit_ens(self, ens): """ Emit the ensemble data to the RabbitMQ. The ensemble data will be pickled using JSON Pickle. :param ens: Ensemble data. """ serial = "0000" if ens.IsEnsembleData: serial = ens.EnsembleData.SerialNumber self.rabbit.send("adcp." + serial + ".data.live", jsonpickle.dumps(ens, unpicklable=False))
class EnsembleFileReader: def __init__(self): self.ens_receiver = None self.ens_reader = None # Codec to decode the data from the file self.codec = AdcpCodec(55057) self.codec.EnsembleEvent += self.process_ensemble_codec self.ens_count = 0 self.ens_codec_count = 0 def process(self, file_path): """ Read the file and start a thread to monitor the incoming ensembles. :param file_path: File path the read files :return: """ # Create ensemble receiver self.ens_receiver = EnsembleReceiver() self.ens_receiver.EnsembleEvent += self.process_ensemble # Start thread to monitor incoming ensembles # Connect to ensemble server self.ens_reader = threading.Thread(name='EnsFileReader', target=self.ens_receiver.connect, args=[55057]).start() # Process the file self.process_file(file_path) # Stop the receiver self.ens_receiver.close() logger.info("Completed File reader") logger.info("Ensemble UDP Count: " + str(self.ens_count)) logger.info("Ensemble Codec Count: " + str(self.ens_codec_count)) def process_file(self, file_path): """ Process the file given. This read from the file and add it to the codec. The codec will then decode the data and pass it to the UDP port. """ # Check if the file exist if os.path.exists(file_path): logger.info("Open file: " + file_path) # Open the file f = open(file_path, "rb") # Add the data from the file to the codec data = f.read(4096) while len(data) > 0: # Add data to codec self.codec.add(data) # Read next block from the file data = f.read(4096) # Close the file f.close() else: logger.error("File does not exist") def process_ensemble(self, sender, ens): """ Receive and process the incoming ensemble from the UDP port. This data has been processed through the codec then passed over the UDP port as JSON data. The JSON datasets were then collected and assembled as a JSON ensemble. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ print("UDP: " + str(ens.EnsembleNumber)) self.ens_count += 1 def process_ensemble_codec(self, sender, ens): """ Receive and process the incoming ensemble directly from the codec. This data was process and passed as an Ensemble object. :param sender: Sender of the ensemble. :param ens: Ensemble data. """ if ens.IsEnsembleData: print("Codec: " + str(ens.EnsembleData.EnsembleNumber)) self.ens_codec_count += 1
class WampSerialProtocol(LineReceiver): """ Serial communication protocol. """ # need a reference to our WS-MCU gateway factory to dispatch PubSub events def __init__(self, session, port, baud): # Get a reference to Application session self.session = session self.serialPort = None #port = self.session.config.extra['port'] #baud = self.session.config.extra['baudrate'] try: self.serialPort = SerialPort(self, port, reactor, baudrate=baud) except Exception as e: self.session.log.error('Could not open serial port: {0}'.format(e)) # Setup codec self.codec = AdcpCodec() self.codec.EnsembleEvent += self.ensemble_event def onConnect(self, response): print("Server connected: {0}".format(response.peer)) self.factory.resetDelay() def connectionMade(self): """ Called when the serial made a connection. :return: """ self.session.log.info('Serial port connected.') def connectionLost(self, reason): """ Called if the connection is lost. If the Connection was lost because the connection was disconnected properly, then do nothing. If it lost connection because the serial port was lost, then try to reconnect. :param reason: :return: """ self.session.log.info("Lost connection (%s)" % reason) # Check if the reason was a good disconnect if reason.value.__class__ != twisted.internet.error.ConnectionDone: self.session.log.info("Reconnecting in 5 seconds...") self.retry = reactor.callLater(5, self.reconnect_attempt) def reconnect_attempt(self): """ Used so that checking for disconnects, it does not need to know the current configuration. :return: """ # Get the port and baud from the configuration port = self.session.config.extra['port'] baud = self.session.config.extra['baudrate'] self.reconnect(port, baud) def reconnect(self, port, baud): """ Reconnect the serial port. :return: """ self.session.log.info("Try to reconnect") # Reset the transport self.transport.loseConnection() # Create a new serial port try: self.serialPort = SerialPort(self, port, reactor, baudrate=baud) except Exception as e: self.session.log.error('Could not open serial port: {0}'.format(e)) self.session.log.info("Reconnecting in 5 seconds...") self.retry = reactor.callLater(5, self.reconnect_attempt) def dataReceived(self, data): """ Data received from the serial port. :param data: Data received from the serial port. :return: """ payload = {} payload["port"] = self.session.config.extra['port'] payload["baud"] = self.session.config.extra['baudrate'] try: payload["value"] = data.decode('utf-8') payload["type"] = "command" except: payload["value"] = str(data) payload["type"] = "binary" # Publish WAMP event to all subscribers on topic self.session.publish(u"com.rti.data.serial", json.dumps(payload)) # Add data to the codec self.codec.add(data) def lineReceived(self, line): # Not Used self.session.log.info("Serial line RX: {0}".format(line)) def ensemble_event(self, sender, ens): """ This is called when the codec has a processed ensemble. :param sender: Sender of the ensemble. :param ens: Ensemble as JSON. :return: """ # publish WAMP event to all subscribers on topic self.session.publish(u"com.rti.data.ens", json.dumps(ens, default=lambda o: o.__dict__)) def send_command(self, cmd): """ Send a command to the serial port. :param cmd: Command to send to the serial port. :return: """ self.session.log.info("Serial TX: {0}".format(cmd)) try: self.transport.write((cmd + "\r").encode('ascii', 'ignore')) except Exception as e: self.session.log.error(str(e)) def send_break(self, time): """ Send a BREAK to the serial port. :param time: Duration of the BREAK. :return: """ self.session.log.info("Serial TX BREAK: {0}".format(str(time))) try: self.transport.sendBreak() #self.serialPort.sendBreak() except Exception as e: self.session.log.error("send_break Error: " + str(e))
class ReadRawSerialThread(QtCore.QThread): """ Create a Read raw serial data from TCP port thread. """ raw_data = QtCore.Signal(object) def __init__(self, tcp_socket, ens_port, parent=None): QtCore.QThread.__init__(self, parent) self.socket = tcp_socket self.isAlive = True logger.debug("Read Socket thread started") # Initialize the ADCP Codec self.codec = AdcpCodec(ens_port) # Setup Waves ens_in_burst = settings.get('WavesProjectSection', 'EnsemblesInBurst') file_path = settings.get('WavesProjectSection', 'CaptureFilePath') lat = settings.get('WavesProjectSection', 'Lat') lon = settings.get('WavesProjectSection', 'Lon') bin1 = settings.get('WavesProjectSection', 'Bin1') bin2 = settings.get('WavesProjectSection', 'Bin2') bin3 = settings.get('WavesProjectSection', 'Bin3') self.codec.enable_waveforce_codec(int(ens_in_burst), file_path, float(lat), float(lon), int(bin1), int(bin2), int(bin3)) def stop(self): """ Stop the thread by setting the isAlive flag. """ self.isAlive = False def run(self): """ Run the loop that views the data from the serial port. :return: """ self.exec() def exec(self): """ Run the loop to view data from the serial port. Emit the data so the view can view the data. """ while self.isAlive: try: # Read data from socket data = self.socket.recv(4096) # If data exist process if len(data) > 0: self.raw_data.emit(data) # Pass data to the decoder self.codec.add(data) except socket.timeout: # Just a socket timeout, continue on pass logger.debug("Read Thread turned off")