def new_waterframe(self):
"""Create a new WaterFrame object and clean all screens."""
self.wf = WaterFrame()
# Delete all plots
for plot_widget in self.plot_widget_list:
plot_widget.deleteLater()
self.plot_widget_list.clear()
# Hide the widget
self.hide()
def __init__(self):
"""
Constructor
"""
super().__init__()
# Instance variables
self.wf = WaterFrame()
# List of PlotWidget, to control them in any case
self.plotWidgetList = []
self.initUI()
def __init__(self):
"""
Constructor
"""
super().__init__()
# Instance variables
self.wf = WaterFrame() # pylint: disable=C0103
# List of PlotWidget, to control them in any case
self.plot_widget_list = []
self.init_ui()
def __init__(self):
super().__init__()
# Instance variables
self.downloader = EGIM()
self.wf = WaterFrame()
self.metadata = dict()
self.dates = []
self.myThread = None
# Save the login of the EMSODEV API
self.downloader.login = "******"
self.downloader.password = ""
self.initUI()
def saveData(self, path):
"""
Save current data into a pickle file
:param path: File path
:return: Bool
"""
self.msg2Statusbar.emit("Saving data")
extension = path.split(".")[-1]
# Init ok
ok = False
wf_new = WaterFrame()
if extension == "nc":
pass
elif extension == "pkl":
ok = self.wf.to_pickle(path)
elif extension == "csv":
ok = self.wf.to_csv(path)
if ok:
self.msg2TextArea.emit("Data saved on file {}".format(path))
self.msg2Statusbar.emit("Ready")
return ok
path_ctd = path_location + r"\CTD.csv"
path_oximeter = path_location + r"\Oximeter.csv"
output_file = r""
# Load data into a WaterFrame
print("Loading data from CTD")
wf_ctd = EGIM.from_raw_csv("EMSO-Azores", path_ctd)
print("Done")
print("Loading data from the Oximeter")
wf_oxymeter = EGIM.from_raw_csv("EMSO-Azores", path_oximeter)
print("Done")
# Creation of a third Waterframe
print("Creation of an other WaterFrame")
wf_anal = WaterFrame()
print("Concat CTD")
wf_anal.concat(wf_ctd)
print("Done")
print("Rename parameters")
wf_anal.rename("TEMP", "CTD")
print("Done")
print("Concat Oximeter")
wf_anal.concat(wf_oxymeter)
print("Done")
print("Rename parameters")
wf_anal.rename("TEMP", "OXYMETER")
print("Done")
# Calculation of correlation between the TEMP parameters
print(wf_anal.corr("CTD", "OXYMETER"))
from mooda import WaterFrame
# Path of files
# Where the files are:
path_location = r"C:\Users\rbard\Desktop\Azores EGIM Data"
# Where to save the result:
save_location = r"C:\Users\rbard\Google Drive\Work\EmsoDev\server\www\html\graphs\azores"
path_files = glob.glob(path_location)
df = []
for path_file in path_files:
# Load data into a WaterFrame
print("Loading data")
wf = WaterFrame()
wf.from_netcdf(path_file)
print("Done")
print(wf)
# Resample data hourly
print("Resampling data")
wf.resample("H")
print("Done")
# Obtain one of the parameters
parameter = wf.parameters()[0]
intervals = wf.datetime_intervals(parameter)
if "R_4381-606" in path_file:
def open_data(self, path, concat=False):
"""
It opens the netcdf of the path.
Parameters
----------
path: str or WaterFrame
Path where the file is or WaterFrame object.
concat: bool, optional (concat = False)
It adds the new dataframe to the current dataframe.
Returns
-------
True/False: bool
It indicates if the operation is ok.
"""
debug = True # Variable for debug reasons
if debug:
print("In PlotSplitter.open_data():")
print(" - Emit msg2statusbar: Opening data")
self.msg2statusbar.emit("Opening data")
if isinstance(path, str):
if debug:
print(" - path is a string:", path)
# Obtain type of file
extension = path.split(".")[-1]
# Init ok
ok = False # pylint: disable=C0103
wf_new = WaterFrame()
if extension == "nc":
ok = wf_new.from_netcdf(path) # pylint: disable=C0103
elif extension == "pkl":
ok = wf_new.from_pickle(path) # pylint: disable=C0103
if ok:
# Check if we want actual data
if not concat:
self.new_waterframe()
self.wf.concat(wf_new)
self.msg2TextArea.emit("Working with file {}".format(path))
# Add metadata information into metadataList
self.add_metadata(self.wf.metadata)
# Add other information
self.other_info_plain_text.setPlainText(repr(self.wf))
# Add data information into data_list
self.add_data(self.wf.data)
# Plot QC
self.add_qc_bar_plot()
self.msg2statusbar.emit("Ready")
if debug:
print(" - Emit msg2statusbar: Ready")
print(" - Exit from PlotSplitter.open_data()")
return True
else:
self.msg2statusbar.emit("Error opening data")
if debug:
print(" - Emit msg2statusbar: Error opening data")
print(" - Exit from PlotSplitter.open_data()")
return False
else:
# Path is a WaterFrame
if debug:
print(" - path is a WaterFrame")
print(path)
# Check if there is no data in the waterframe
if path.data.empty:
return False
# Check if it is a dataframe of an acoustic data.
# In this case, we are going to delete the previous dataframe.
if "Sequence" in self.wf.data.keys():
self.wf.clear()
self.wf.concat(path)
# Add data information into data_list
self.add_data(self.wf.data)
self.add_metadata(self.wf.metadata)
# Add other information
self.other_info_plain_text.setPlainText(repr(self.wf))
# Plot QC
self.add_qc_bar_plot()
return True
class PlotSplitter(QSplitter):
"""Pyqt5 widget to show data in plots and lists."""
# Signals
msg2statusbar = pyqtSignal(str)
msg2TextArea = pyqtSignal(str)
def __init__(self):
"""
Constructor
"""
super().__init__()
# Instance variables
self.wf = WaterFrame() # pylint: disable=C0103
# List of PlotWidget, to control them in any case
self.plot_widget_list = []
self.init_ui()
def init_ui(self):
"""UI creator"""
# Lists
self.data_list = QListWidget(self)
self.data_list.setSelectionMode(QAbstractItemView.ExtendedSelection)
self.data_list.itemClicked.connect(self.data_list_click)
self.graph_list = QListWidget(self)
self.graph_list.itemClicked.connect(self.graph_click)
# PlainTextEdit
self.other_info_plain_text = QPlainTextEdit(self)
self.metadata_plain_text = QPlainTextEdit(self)
# Labels
metadata_label = QLabel("Metadata")
data_label = QLabel("Data")
graph_label = QLabel("Graph")
info_label = QLabel("Other information")
# DropWidget
self.drop_widget = DropWidget()
self.drop_widget.list2drop[list, list, bool].connect(self.drop_data)
self.drop_widget.hide()
# Group Box
plot_group_box = QGroupBox("Plot properties", self)
v_plot_group_box = QVBoxLayout()
# - RadioButton
self.auto_plot_radio_button = QRadioButton("Time series plot", self)
self.auto_plot_radio_button.setChecked(True)
self.multiple_parameter_radio_button = QRadioButton(
"Multiparameter", self)
self.correlation_radio_button = QRadioButton("Correlation", self)
self.histogram_radio_button = QRadioButton("Histogram", self)
self.parameter_qc_radio_button = QRadioButton("QC of the parameter",
self)
v_plot_group_box.addWidget(self.auto_plot_radio_button)
v_plot_group_box.addWidget(self.histogram_radio_button)
v_plot_group_box.addWidget(self.multiple_parameter_radio_button)
v_plot_group_box.addWidget(self.correlation_radio_button)
v_plot_group_box.addWidget(self.parameter_qc_radio_button)
plot_group_box.setLayout(v_plot_group_box)
# QCWidget
self.qc_widget = QCWidget()
self.qc_widget.list2qc[list].connect(self.apply_qc)
self.qc_widget.hide()
# RenameWidget
self.rename_widget = RenameWidget()
self.rename_widget.key2change[str, str].connect(self.apply_rename)
self.rename_widget.hide()
# ResampleWidget
self.resample_widget = ResampleWidget()
self.resample_widget.resampleRule[str].connect(self.apply_resample)
self.resample_widget.hide()
# SliceWidget
self.slice_widget = SliceWidget()
self.slice_widget.sliceTimes[str, str].connect(self.apply_slice)
self.slice_widget.hide()
# Splitters
self.v_data_splitter = QSplitter(Qt.Vertical)
# Custom Widget Metadata
self.v_metadata_widget = QWidget()
# Buttons
# - For metadata area
hide_metadata_button = QPushButton("Hide")
hide_metadata_button.clicked.connect(self.v_metadata_widget.hide)
# - For data Area
plot_button = QPushButton("Plot")
plot_button.clicked.connect(self.add_plot)
hide_data_button = QPushButton("Hide")
hide_data_button.clicked.connect(self.v_data_splitter.hide)
# Custom Widget Data
# - Data Widget
parameter_widget = QWidget()
# -- Layout
v_data = QVBoxLayout()
v_data.addWidget(data_label)
v_data.addWidget(self.data_list)
v_data.addWidget(plot_group_box)
v_data.addWidget(plot_button)
parameter_widget.setLayout(v_data)
# - Graph Widget
graph_widget = QWidget()
# -- Layout
v_graph = QVBoxLayout()
v_graph.addWidget(graph_label)
v_graph.addWidget(self.graph_list)
v_graph.addWidget(hide_data_button)
graph_widget.setLayout(v_graph)
# - Data splitter
self.v_data_splitter.addWidget(parameter_widget)
self.v_data_splitter.addWidget(graph_widget)
# Layouts
# - Metadata -
v_metadata = QVBoxLayout()
v_metadata.addWidget(metadata_label)
v_metadata.addWidget(self.metadata_plain_text)
v_metadata.addWidget(info_label)
v_metadata.addWidget(self.other_info_plain_text)
v_metadata.addWidget(hide_metadata_button)
self.v_metadata_widget.setLayout(v_metadata)
# Splitter (self)
# - Layout for actions
v_actions_widget = QWidget()
v_actions = QVBoxLayout()
v_actions.addWidget(self.rename_widget)
v_actions.addWidget(self.resample_widget)
v_actions.addWidget(self.drop_widget)
v_actions.addWidget(self.slice_widget)
v_actions.addStretch()
v_actions_widget.setLayout(v_actions)
# - Add to self (splitter)
self.addWidget(self.qc_widget)
self.addWidget(v_actions_widget)
self.addWidget(self.v_metadata_widget)
self.addWidget(self.v_data_splitter)
def data_list_click(self):
"""Action on lick item of data_list"""
if (self.auto_plot_radio_button.isChecked()
or self.histogram_radio_button.isChecked()):
self.add_plot()
def add_plot(self):
"""It creates a FigureCanvas with the input figure"""
self.msg2statusbar.emit("Making the figure")
# Create key list
keys = [item.text() for item in self.data_list.selectedItems()]
# If nothing is selected, go out
if not keys:
self.msg2statusbar.emit("Ready")
return
# Check if it is a QC plot
if self.parameter_qc_radio_button.isChecked():
keys = [keys[0] + "_QC"]
# Check if it is a correlation
if self.correlation_radio_button.isChecked():
plot_widget = ScatterMatrixPlotWidget(wf=self.wf, keys=keys)
self.addWidget(plot_widget)
self.msg2statusbar.emit("Ready")
return
# Create name of the plot
name = '_'.join(keys)
if self.histogram_radio_button.isChecked():
name = "hist_" + name
# Check if plot is done
new = True
for plot_widget in self.plot_widget_list:
if plot_widget.name == name:
if ~plot_widget.isVisible():
plot_widget.refresh_plot()
plot_widget.show()
new = False
break
# Create the plot if is new
if new:
if len(keys) == 1 and "_QC" in keys[0]:
plot_widget = QCPlotWidget(wf=self.wf, key=keys[0])
else:
if self.histogram_radio_button.isChecked():
plot_widget = HistoPlotWidget(wf=self.wf, keys=keys)
else:
plot_widget = TSPlotWidget(wf=self.wf, keys=keys)
plot_widget.msg2statusbar[str].connect(self.msg2statusbar.emit)
self.addWidget(plot_widget)
# Add the widget to the list
self.plot_widget_list.append(plot_widget)
self.msg2statusbar.emit("Ready")
def add_qc_bar_plot(self):
"""
It creates a FigureCanvas with the input figure (QC)
"""
self.msg2statusbar.emit("Making the figure")
# Check if the plot exists
plot_widget = None
for plot_widget_ in self.plot_widget_list:
if plot_widget_.name == "QC":
plot_widget = plot_widget_
plot_widget.wf = self.wf
plot_widget.refresh_plot()
plot_widget.show()
break
if plot_widget is None:
plot_widget = QCBarPlotWidget(wf=self.wf)
self.addWidget(plot_widget)
# Add the widget to the list
self.plot_widget_list.append(plot_widget)
self.msg2statusbar.emit("Ready")
def add_spectro_plot(self):
"""It ads the SpectroplotWidget to the screen"""
self.msg2statusbar.emit("Making the figure")
# Check if the plot exists
plot_widget = None
for plot_widget_ in self.plot_widget_list:
if plot_widget_.name == "Spectrogram":
plot_widget = plot_widget_
plot_widget.wf = self.wf
plot_widget.refresh_plot()
plot_widget.show()
break
if plot_widget is None:
plot_widget = SpectrogramPlotWidget(wf=self.wf)
self.addWidget(plot_widget)
# Add the widget to the list
self.plot_widget_list.append(plot_widget)
self.msg2statusbar.emit("Ready")
def open_data(self, path, concat=False):
"""
It opens the netcdf of the path.
Parameters
----------
path: str or WaterFrame
Path where the file is or WaterFrame object.
concat: bool, optional (concat = False)
It adds the new dataframe to the current dataframe.
Returns
-------
True/False: bool
It indicates if the operation is ok.
"""
debug = True # Variable for debug reasons
if debug:
print("In PlotSplitter.open_data():")
print(" - Emit msg2statusbar: Opening data")
self.msg2statusbar.emit("Opening data")
if isinstance(path, str):
if debug:
print(" - path is a string:", path)
# Obtain type of file
extension = path.split(".")[-1]
# Init ok
ok = False # pylint: disable=C0103
wf_new = WaterFrame()
if extension == "nc":
ok = wf_new.from_netcdf(path) # pylint: disable=C0103
elif extension == "pkl":
ok = wf_new.from_pickle(path) # pylint: disable=C0103
if ok:
# Check if we want actual data
if not concat:
self.new_waterframe()
self.wf.concat(wf_new)
self.msg2TextArea.emit("Working with file {}".format(path))
# Add metadata information into metadataList
self.add_metadata(self.wf.metadata)
# Add other information
self.other_info_plain_text.setPlainText(repr(self.wf))
# Add data information into data_list
self.add_data(self.wf.data)
# Plot QC
self.add_qc_bar_plot()
self.msg2statusbar.emit("Ready")
if debug:
print(" - Emit msg2statusbar: Ready")
print(" - Exit from PlotSplitter.open_data()")
return True
else:
self.msg2statusbar.emit("Error opening data")
if debug:
print(" - Emit msg2statusbar: Error opening data")
print(" - Exit from PlotSplitter.open_data()")
return False
else:
# Path is a WaterFrame
if debug:
print(" - path is a WaterFrame")
print(path)
# Check if there is no data in the waterframe
if path.data.empty:
return False
# Check if it is a dataframe of an acoustic data.
# In this case, we are going to delete the previous dataframe.
if "Sequence" in self.wf.data.keys():
self.wf.clear()
self.wf.concat(path)
# Add data information into data_list
self.add_data(self.wf.data)
self.add_metadata(self.wf.metadata)
# Add other information
self.other_info_plain_text.setPlainText(repr(self.wf))
# Plot QC
self.add_qc_bar_plot()
return True
def add_metadata(self, metadata_dict):
"""
Add Metadata information into self.metadata_plain_text
:param metadata_dict: WaterFrame Metadata Dictionary
"""
# Clear the list
self.metadata_plain_text.clear()
items = []
msg = "\nMetadata:"
for key, value in metadata_dict.items():
items.append("{}: {}".format(key, value))
msg += "\n- {}: {}".format(key, value)
self.metadata_plain_text.setPlainText(msg[11:])
# Send a message to the text area
self.msg2TextArea.emit(msg)
def add_data(self, data):
"""
Add data names into self.data_list
:param data: WaterFrame data variable
"""
def is_acoustic_data(a): # pylint: disable=C0103
# will be True also for 'NaN'
if a == "Sequence":
return True
elif a == "Data Points":
return True
try:
float(a)
return True
except ValueError:
return False
# Clear the list
self.data_list.clear()
# Parameter keys (without QC)
# NO DEPTH in nc files, NO TIME
keys_to_work = [
key for key in data.keys() if 'TIME' not in key
if not is_acoustic_data(key) if key + "_QC" in data.keys()
]
self.data_list.addItems(keys_to_work)
# Add graphs
self.graph_list.clear()
self.graph_list.addItem("QC")
# Check if we have acoustic data
for key in data.keys():
if is_acoustic_data(key):
self.graph_list.addItem("Spectrogram")
break
# Add tooltip
msg = "\nData:"
for i in range(self.data_list.count()):
if "_QC" in self.data_list.item(i).text():
self.data_list.item(i).setToolTip('QC flags of {}'.format(
self.data_list.item(i).text()[:-3]))
else:
try:
self.data_list.item(i).setToolTip('{} ({})'.format(
self.wf.meaning[self.data_list.item(i).text()]
['long_name'], self.wf.meaning[self.data_list.item(
i).text()]['units']))
msg += "\n- {}: {} ({})".format(
self.data_list.item(i).text(), self.wf.meaning[
self.data_list.item(i).text()]['long_name'],
self.wf.meaning[self.data_list.item(
i).text()]['units'])
except KeyError:
pass
# Send a message to the text area
self.msg2TextArea.emit(msg)
# Send the labels to the drop_widget
self.drop_widget.add_labels(keys_to_work)
self.qc_widget.add_labels(keys_to_work)
self.rename_widget.add_labels(keys_to_work)
self.slice_widget.refresh(self.wf.data.index[0],
self.wf.data.index[-1])
def save_data(self, path):
"""
Save current data into a pickle file
:param path: File path
:return: Bool
"""
self.msg2statusbar.emit("Saving data")
extension = path.split(".")[-1]
# Init ok
ok = False # pylint: disable=C0103
if extension == "nc":
pass
elif extension == "pkl":
ok = self.wf.to_pickle(path) # pylint: disable=C0103
elif extension == "csv":
ok = self.wf.to_csv(path) # pylint: disable=C0103
if ok:
self.msg2TextArea.emit("Data saved on file {}".format(path))
self.msg2statusbar.emit("Ready")
return ok
def drop_data(self, labels, flag_list, drop_nan):
"""
Delete some parameters from self.wf and refresh the lists
:param labels: list of labels to drop
:param flag_list: list of flags to drop
:return:
"""
self.msg2TextArea.emit("Deleting data")
# This is a trick, delete the list if is a list of None
if flag_list[0] is None:
flag_list = None
if flag_list:
self.wf.use_only(parameters=labels, flags=[0, 1], dropnan=drop_nan)
else:
# Delete the parameters
self.wf.drop(keys=labels, flags=flag_list)
# Refresh the lists
self.add_data(self.wf.data)
# Delete plots with the key
for label in labels:
for plot_widget in self.plot_widget_list:
if plot_widget.name == "QC":
plot_widget.refresh_plot()
if label == plot_widget.name or label+"_" in \
plot_widget.name or "_"+label in plot_widget.name:
plot_widget.deleteLater()
self.plot_widget_list.remove(plot_widget)
# Send message
msg = ""
if flag_list is None:
for label in labels:
if '_QC' in label:
continue
msg += "{} ".format(label)
msg += "deleted"
else:
msg += "Data with QC Flags "
for flag in flag_list:
msg += "{}, ".format(flag)
msg += "from "
for label in labels:
if '_QC' in label:
continue
msg += "{}, ".format(label)
msg += "deleted"
self.msg2TextArea.emit("\n{}".format(msg))
self.msg2statusbar.emit(msg)
def apply_qc(self, list_qc):
"""
Apply the QC procedures
:param list_qc:
:return:
"""
def do_it(key_in):
"""
Common part, to not repeat code
:param key_in: key to apply QC tests
"""
if '_QC' in key_in:
return
if list_qc[0]:
# Reset flags
self.msg2statusbar.emit("Setting flags from {} to {}".format(
key_in, list_qc[0]))
self.wf.reset_flag(parameters=key_in, flag=int(list_qc[0]))
self.msg2statusbar.emit("Ready")
if list_qc[3]:
# Spike test
threshold = float(list_qc[4].replace(',', '.'))
self.msg2statusbar.emit(
"Applying spike test to "
"{}, with rolling window {} and threshold {}".format(
key_in, list_qc[5], threshold))
self.wf.spike_test(parameters=key_in,
window=int(list_qc[5]),
threshold=threshold,
flag=int(list_qc[3]))
self.msg2statusbar.emit("Ready")
if list_qc[1]:
# Range test
self.msg2statusbar.emit(
"Applying range test to {}".format(key_in))
self.wf.range_test(parameters=key_in, flag=int(list_qc[1]))
self.msg2statusbar.emit("Ready")
if list_qc[2]:
# Flat test
self.msg2statusbar.emit("Applying flat test to"
" {}, with rolling window {}".format(
key_in, list_qc[5]))
self.wf.flat_test(parameters=key_in,
window=int(list_qc[5]),
flag=int(list_qc[2]))
self.msg2statusbar.emit("Ready")
if list_qc[6]:
# Flag to flag
self.msg2statusbar.emit("Changing flags of "
"{} from {} to {}".format(
key_in, list_qc[6], list_qc[7]))
self.wf.flag2flag(parameters=key_in,
original_flag=int(list_qc[6]),
translated_flag=int(list_qc[7]))
self.msg2statusbar.emit("Ready")
self.msg2statusbar.emit("Creating QC flags")
if list_qc[8] == 'all':
for key in self.wf.parameters():
do_it(key_in=key)
else:
for i in range(8, len(list_qc)):
key = list_qc[i]
do_it(key_in=key)
self.msg2statusbar.emit("Updating graphs")
# Refresh the QC graph
for plot_widget in self.plot_widget_list:
if "QC" in plot_widget.name:
if plot_widget.isVisible():
plot_widget.refresh_plot()
# Show the QCBarPlot
elif plot_widget.name == "QC":
plot_widget.refresh_plot()
plot_widget.show()
self.msg2statusbar.emit("Ready")
def apply_rename(self, original_key, new_key):
"""It renames keys from a WaterFrame"""
# Rename key from the Waterframe
self.msg2statusbar.emit("Changing name {} to {}".format(
original_key, new_key))
self.wf.rename(original_key, new_key)
# Rename the key of the plotWidgets if it process
for plot_widget in self.plot_widget_list:
if isinstance(plot_widget.key, list):
for i, key in enumerate(plot_widget.key):
if key == original_key:
plot_widget.key[i] = new_key
plot_widget.name = plot_widget.name.replace(
original_key, new_key)
plot_widget.refresh_plot()
else:
if plot_widget.name == "QC":
plot_widget.refresh_plot()
elif plot_widget.key == original_key:
plot_widget.key = new_key
plot_widget.name = new_key
plot_widget.refresh_plot()
# Add data information into data_list
self.add_data(self.wf.data)
self.msg2statusbar.emit("Ready")
self.msg2TextArea.emit("Key name {} changed to {}.".format(
original_key, new_key))
def apply_resample(self, rule):
"""
It applies the resample function to self.waterframe
:param rule: Rule to resample.
"""
self.msg2statusbar.emit("Resampling data")
self.wf.resample(rule)
self.msg2statusbar.emit("Ready")
self.msg2statusbar.emit("Updating graphs")
# Refresh the QC graph
for plot_widget in self.plot_widget_list:
if "QC" in plot_widget.name:
if plot_widget.isVisible():
plot_widget.refresh_plot()
# Show the QCBarPlot
elif plot_widget.name == "QC":
plot_widget.refresh_plot()
plot_widget.show()
self.msg2statusbar.emit("Ready")
def apply_slice(self, start, stop):
"""
It applies the resample function to self.waterframe
:param start: Start time.
:param stop: Stop time
"""
self.msg2statusbar.emit("Slicing data")
self.wf.slice_time(start, stop)
self.add_data(self.wf.data)
self.refresh_plots()
self.msg2statusbar.emit("Ready")
self.msg2TextArea.emit("Dataframe sliced from {} to {}.".format(
start, stop))
def graph_click(self, item):
"""Function on click the plot button"""
if item.text() == "QC":
self.add_qc_bar_plot()
elif item.text() == "Spectrogram":
self.add_spectro_plot()
def new_waterframe(self):
"""Create a new WaterFrame object and clean all screens."""
self.wf = WaterFrame()
# Delete all plots
for plot_widget in self.plot_widget_list:
plot_widget.deleteLater()
self.plot_widget_list.clear()
# Hide the widget
self.hide()
def refresh_plots(self):
"""It refresh all plots"""
self.msg2statusbar.emit("Refreshing plots")
for plot_widget in self.plot_widget_list:
try:
plot_widget.refresh_plot()
except KeyError:
self.plot_widget_list.remove(plot_widget)
plot_widget.hide()
plot_widget.deleteLater()
self.msg2statusbar.emit("Ready")
from mooda import WaterFrame
directory_path = r"PATH"
file_path = r"FILE"
path = directory_path + file_path
print("Loading WaterFrame")
wf = WaterFrame(path)
print("Done")
save_path = "{}csv".format(path[:-2])
print("Saving to CSV")
wf.to_csv(save_path)
print("Done")
def openData(self, path, concat=False):
"""
It opens the netcdf of the path.
Parameters
----------
path: str or WaterFrame
Path where the file is or WaterFrame object.
concat: bool, optional (concat = False)
It adds the new dataframe to the current dataframe.
Returns
-------
True/False: bool
It indicates if the operation is ok.
"""
self.msg2Statusbar.emit("Opening data")
if isinstance(path, str):
# Obtain type of file
extension = path.split(".")[-1]
# Init ok
ok = False
wf_new = WaterFrame()
if extension == "nc":
ok = wf_new.from_netcdf(path)
elif extension == "pkl":
ok = wf_new.from_pickle(path)
elif extension == "csv":
try:
wf_new = EGIM.from_raw_csv("EMSO-Azores", path)
ok = True
except:
ok = False
if ok:
# Check if we want actual data
if not concat:
self.newWaterFrame()
self.wf.concat(wf_new)
self.msg2TextArea.emit("Working with file {}".format(path))
# Add metadata information into metadataList
self.addMetadata(self.wf.metadata)
# Add other information
self.otherInfoPlainText.setPlainText(repr(self.wf))
# Add data information into dataList
self.addData(self.wf.data)
# Plot QC
self.addQCBarPlot()
self.msg2Statusbar.emit("Ready")
return True
else:
self.msg2Statusbar.emit("Error opening data")
return False
else:
# Path is a WaterFrame
# Check if there is no data in the waterframe
if path.data.empty:
return False
# Check if it is a dataframe of an acoustic data.
# In this case, we are going to delete the previous dataframe.
if "Sequence" in self.wf.data.keys():
self.wf.clear()
self.wf.concat(path)
# Add data information into dataList
self.addData(self.wf.data)
self.addMetadata(self.wf.metadata)
# Plot QC
self.addQCBarPlot()
return True
def from_txt(model, path, qc_tests=False):
"""
Parameters
----------
model: str
Model of the instrument.
path: str
Path of the txt file.
qc_tests: bool (optional)
It indicates if QC test should be passed.
Returns
-------
wf: WaterFrame
"""
# Creation of a WaterFrame
wf = WaterFrame()
metadata = {}
# The arguments of pandas.read_csv could change depending on the
# source.
if model == "LI-192":
# lines to skip (int) at the start of the file.
skiprows = 6
# format_time = '%d/%m/%Y %H:%M:%S'
# Add metadata info
with open(path, encoding='utf-8-sig') as txtfile:
# Read 2 first lines from csv
for row in itertools.islice(txtfile, 1, 6):
# Delete return carriage from row
row = ''.join(row.splitlines())
# Split row by "\t" and remove empty strings returned in
# split()
parts = row.split(":")
if "Timestamp" not in parts[0]:
metadata[parts[0]] = parts[1].strip()
# Load metadata to waterframe
wf.metadata = metadata
# Load data from table into a DataFrame
df = pd.read_table(path,
delim_whitespace=True,
skiprows=skiprows,
low_memory=False)
df['Ns'] = pd.to_numeric(df['Nanoseconds'],
errors='coerce') / 1000000000
df['Input1'] = pd.to_numeric(df['Input1'], errors='coerce')
# Create the time index. Convert "Nanoseconds" to seconds, add to
# column "Seconds"
# and convert to datetime
df['Seconds_total'] = df['Ns'] + pd.to_numeric(df['Seconds'],
errors='coerce')
df['TIME'] = pd.to_datetime(df['Seconds_total'],
unit='s',
errors='coerce')
df.set_index(df['TIME'], inplace=True)
df.drop([
'DATAH', 'Record', 'Seconds', 'Nanoseconds', 'Ns',
'Seconds_total', 'TIME', 'MULT_1', 'CHK'
],
inplace=True,
axis=1)
# Add DataFrame into the WaterFrame
wf.data = df.copy()
# Change parameter names and add QC columns
for key in wf.data.keys():
if key == "Input1":
wf.data.rename(columns={"Input1": "PPFD"}, inplace=True)
wf.data["PPFD_QC"] = 0
wf.meaning['PPFD'] = {
"long_name": "Photosynthetic Photon Flux Density",
"units": "µMol/M^2S"
}
# Creation of QC Flags following OceanSites recomendation
if qc_tests:
for parameter in wf.parameters():
# Reset QC Flags to 0
wf.reset_flag(key=parameter, flag=0)
# Flat test
wf.flat_test(key=parameter, window=0, flag=4)
# Spike test
wf.spike_test(key=parameter, window=0, threshold=3, flag=4)
# Range test
wf.range_test(key=parameter, flag=4)
# Change flags from 0 to 1
wf.flag2flag(key=parameter,
original_flag=0,
translated_flag=1)
else:
warnings.warn("Unknown model")
return
# resample to seconds
wf.resample('S')
return wf
i_flat = 0
for i in x:
if i_flat < flat_repeat:
data[i] = data[i - 1]
i_flat += 1
else:
i_flat = 0
flat_repeat = 0
flat_bool = np.random.randn()
if flat_bool > 3.8:
flat_repeat = flat_bool * 1000
print(flat_repeat)
df = pd.DataFrame(data, index=dates, columns=["TEMP"])
df.index.name = 'TIME'
df['TEMP_QC'] = 0
# Creation of WaterFrame
wf = WaterFrame()
wf.data = df.copy()
wf.metadata["name"] = "Test data with errors"
wf.meaning["TEMP"] = "Seawater temperature"
units = {'units': 'degree Celsius'}
wf.meaning['TEMP'] = units
wf.to_pickle('test_errors.pkl')
df.plot()
plt.show()
from mooda.access import EGIM
# Path of files
# Where the files are:
path_location = r"C:\Users\rbard\Desktop\Azores EGIM Data"
path_data = path_location + r"\turbidity.csv"
path_data = r"C:\Users\rbard\Desktop\Azores EGIM Data\Oximeter.csv"
# Where to save the result:
save_location = r"C:\Users\rbard\Google Drive\Work\EmsoDev\server\www\html\graphs\azores"
save_start = r"\OXI-"
end_save = r"-AZORES.html"
# Load data into a WaterFrame
print("Loading data")
if path_data[-1] == 'c':
wf = WaterFrame()
wf.from_netcdf(path_data)
elif path_data[-1] == 'v':
print(path_data)
wf = EGIM.from_raw_csv(observatory="EMSO-Azores", path=path_data)
print("Done")
print(wf)
# Check parameters
parameters = wf.parameters()
# Resample data hourly
print("Resampling data")
wf.resample("H")
print("Done")
def from_csv(self, qc_tests=False):
"""
It opens a csv file that contains data from an HOBO instrument.
Parameters
----------
model: str
Model of the instrument.
qc_tests: bool (optional)
It indicates if QC test should be passed.
Returns
-------
wf: WaterFrame """
# open file
self.f = open(self.path, mode='rt', encoding='utf-8-sig')
# declare metadata variables
self.title, self.sn = None, None
# declare index variables
self.timestamp = None
# declare dict to save name, long_name and units
self.name = {
'temp': [None, None, None],
'pres': [None, None, None],
'rh': [None, None, None],
'batt': [None, None, None]
}
def find_units(header):
header_units = header.split(",", 2)[1].strip()
units = header_units.split(" ")[0].strip()
return units
def find_name(header):
header_name = header.rsplit(",", 1)[1].strip(" )")
if header_name.split(":")[0].strip() == "LBL":
name = header_name.split(":")[1].strip()
else:
name = header.split(",", 1)[0].strip()
return name
def find_long_name(header):
long_name = header.split(",", 1)[0].strip()
return long_name
def find_col_timestamp(headers):
for i, header in enumerate(headers):
if 'Date Time' in header or 'Fecha Tiempo' in header:
return i
def find_col_temperature(headers):
for i, header in enumerate(headers):
if 'High Res. Temp.' in header or 'High-Res Temp' in header:
self.name['temp'] = [
find_name(header),
find_long_name(header),
find_units(header)
]
# self.temp_units = find_units(header)
# self.temp = find_name(header)
# return 'TEMP'
for i, header in enumerate(headers):
for s in ('Temp,', 'Temp.', 'Temperature'):
if s in header:
self.name['temp'] = [
find_name(header),
find_long_name(header),
find_units(header)
]
# self.temp_units = find_units(header)
# self.temp = find_name(header)
# return 'TEMP'
def find_col_preassure(headers):
for i, header in enumerate(headers):
if 'Pres abs,' in header:
self.name['pres'] = [
find_name(header),
find_long_name(header),
find_units(header)
]
# self.pres = find_units(header)
# self.pres = find_name(header)
# return 'PRES'
def find_col_rh(headers):
for i, header in enumerate(headers):
if 'RH,' in header:
self.name['rh'] = [
find_name(header),
find_long_name(header),
find_units(header)
]
# return 'RH'
def find_col_battery(headers):
for i, header in enumerate(headers):
if 'Batt, V' in header:
self.name['batt'] = [
find_name(header),
find_long_name(header),
find_units(header)
]
# return 'BATT'
def find_columns(header):
""" Find and set column names for headers """
headers = next(csv.reader(StringIO(header)))
self.headers = headers
self.timestamp = find_col_timestamp(headers)
find_col_temperature(headers)
find_col_preassure(headers)
find_col_rh(headers)
find_col_battery(headers)
def find_headers():
while self.timestamp is None:
header = next(self.f)
if self.title is None:
self.title = header.strip().split(":")
if self.sn is None:
sn_match = SN_REGEX.search(header)
self.sn = sn_match.groups()[0] if sn_match else None
find_columns(header)
return header
# find headers
find_headers()
# Creation of a WaterFrame
wf = WaterFrame()
metadata = {}
metadata[self.title[0]] = self.title[1].strip()
metadata["S/N"] = self.sn.strip()
# Load metadata to Waterframe
wf.metadata = metadata
# Create dataframe from csv
df = pd.read_csv(self.f, names=self.headers, index_col=self.timestamp)
df = df.replace(np.nan, '', regex=True)
# Set index to datetime
df.index = pd.to_datetime(df.index)
df.set_index(pd.DatetimeIndex(df.index, inplace=True))
# Rename index
df.index.name = 'Time'
# Rename columns
for col in df.columns:
if 'Temp' in col:
df.rename(columns={col: self.name['temp'][0]}, inplace=True)
elif 'Pres' in col:
df.rename(columns={col: self.name['pres'][0]}, inplace=True)
elif 'Batt' in col:
df.rename(columns={col: self.name['batt'][0]}, inplace=True)
elif 'RH' in col:
df.rename(columns={col: self.name['rh'][0]}, inplace=True)
# Filter columns only if they are present
df = df.filter(items=[
self.name['temp'][0], self.name['pres'][0], self.name['batt'][0],
self.name['rh'][0]
])
# Add QC keys
for key in df.keys():
df["{}_QC".format(key)] = 0
# Add DataFrame into the WaterFrame
wf.data = df.copy()
# Change parameter names and add QC columns
for parameter in wf.parameters():
for key, value in self.name.items():
if value[0] == parameter:
wf.meaning[parameter] = {
"long_name": value[1],
"units": value[2]
}
# Creation of QC Flags following OceanSites recomendation
if qc_tests:
for parameter in wf.parameters():
# Reset QC Flags to 0
wf.reset_flag(key=parameter, flag=0)
# Flat test
wf.flat_test(key=parameter, window=0, flag=4)
# Spike test
wf.spike_test(key=parameter, window=0, threshold=3, flag=4)
# Range test
wf.range_test(key=parameter, flag=4)
# Change flags from 0 to 1
wf.flag2flag(key=parameter, original_flag=0, translated_flag=1)
print(wf.data)
print(wf)
return wf
from mooda import WaterFrame
# Path where CTD file is
path = r"C:\Users\rbard\Google Drive\Work\EmsoDev\server\ejemplos web\megakit - mio\data\obsea\OS_OBSEA_2016120120170426_R_4381-606.nc"
wf = WaterFrame()
wf.from_netcdf(path)
print(wf)
# Resample 2 hourly
# wf.resample('2H')
# Slice
# wf.slice_time("20170327000000", "20170423000000")
mean = wf.mean('DOX2')
index, max_waves = wf.max('DOX2')
index2, min_waves = wf.min('DOX2')
print(index)
print(index2)
# print(max_waves)
# print(min_waves)
# print("Marea:", (max_waves - min_waves)/2)
from mooda import WaterFrame
import matplotlib.pyplot as plt
import matplotlib.style as style
style.use('ggplot')
path = r"./docs/example_data/bad_temp.pkl"
wf = WaterFrame()
wf.from_pickle(path)
print(wf.parameters())
wf.reset_flag(key='TEMP')
wf.qcplot('TEMP')
plt.show()
wf.spike_test(key='TEMP', flag=4)
wf.range_test(key='TEMP', flag=2)
wf.flat_test(key='TEMP', flag=3)
wf.flag2flag(key="TEMP")
wf.qcplot('TEMP')
plt.show()
wf.qcbarplot('TEMP')
plt.show()