def setup(self): conn_dict = {'engine': 'sqlite', 'address': ':memory:'} self.sm = ServiceManager(conn_dict=conn_dict) # set up remote Database # self.connection_string = "sqlite:///:memory:" self.series_service = self.sm.get_series_service( ) # SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.dvs_size = 100 self.series = test_util.add_series_bulk_data(self.session, dvs_size=self.dvs_size) assert self.series assert len(self.series.data_values) == self.dvs_size self.memory_database = MemoryDatabase() self.memory_database.set_series_service(self.series_service) # self.memory_database.initEditValues(self.series.id) self.app = wx.App() self.frame = wx.Frame(None) self.wizard = wizSave( self.frame, self.sm, self.sm.get_edit_service(self.series.id, self.memory_database))
def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService( connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = test_util.add_series(self.session) self.memory_db.initEditValues(self.series.id)
def __init__(self, series_id, connection=None, connection_string="", debug=False): ''' :param series_id: :param connection: memory database, contains connection to remote database :param connection_string: connection to remote database :param debug: :return: ''' self._series_id = series_id self._filter_from_selection = False self._debug = debug if connection_string is "" and connection is not None: self.memDB= connection elif connection_string is not "" and connection is None: from odmtools.odmdata import MemoryDatabase self.memDB= MemoryDatabase() self.memDB.set_series_service(SeriesService(connection_string, False)) else: logger.error("must send in either a remote db connection string or a memory database object") logger.debug("Initializing Memory Database") self.memDB.initEditValues(series_id) logger.debug("Finished Initializing Memory Database") self._populate_series() self.reset_filter()
class TestMemoryDB: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.series = test_util.add_series(self.session) self.memory_db = MemoryDatabase(self.series_service) def test_delete_points(self): with pytest.raises(NotImplementedError): self.memory_db.delete_points("filter") def test_add_points(self): with pytest.raises(NotImplementedError): self.memory_db.add_points("filter") def test_update_points(self): with pytest.raises(NotImplementedError): self.memory_db.update_points("filter", [1, 2, 3]) def test_get_data_values_is_empty(self): dvs = self.memory_db.get_data_values() assert len(dvs) == 0
def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.series = test_util.add_series(self.session) self.memory_db = MemoryDatabase(self.series_service)
class TestPlotProbability: def setup(self): self.app = wx.App() self.frame = wx.Frame(None) self.pltProb = plotProbability.plotProb(id=wxID_PAGEPROB, name='pltProb', parent=self.frame, pos=wx.Point(0, 0), size=wx.Size(605, 458), style=wx.TAB_TRAVERSAL) self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService( connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = add_series(self.session) print "Series: ", self.series self.memory_db.initEditValues(self.series.id) # add_bulk_data_values(self.session, self.series) def test_onPlotType(self): assert self.app assert self.frame assert self.pltProb # add some values to the probability plot valid_values = ['line', 'both', 'point', 'test'] try: for i in valid_values: self.pltProb.onPlotType(ptype=i) except Exception as e: print e assert False
class TestPlotProbability: def setup(self): self.app = wx.App() self.frame = wx.Frame(None) self.pltProb = plotProbability.plotProb(id=wxID_PAGEPROB, name='pltProb', parent=self.frame, pos=wx.Point(0, 0), size=wx.Size(605, 458), style=wx.TAB_TRAVERSAL) self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = add_series(self.session) print "Series: ", self.series self.memory_db.initEditValues(self.series.id) # add_bulk_data_values(self.session, self.series) def test_onPlotType(self): assert self.app assert self.frame assert self.pltProb # add some values to the probability plot valid_values = ['line', 'both', 'point', 'test'] try: for i in valid_values: self.pltProb.onPlotType(ptype=i) except Exception as e: print e assert False
def __init__(self, series_id, connection=None, connection_string="", debug=False): ''' :param series_id: :param connection: memory database, contains connection to remote database :param connection_string: connection to remote database :param debug: :return: ''' self._series_id = series_id self._filter_from_selection = False self._debug = debug if connection_string is "" and connection is not None: self.memDB = connection #self._series_service = self.memDB.series_service#SeriesService(connection_string, debug) elif connection_string is not "" and connection is None: from odmtools.odmdata import MemoryDatabase self.memDB = MemoryDatabase() #(series_service) self.memDB.set_series_service( SeriesService(connection_string, False)) else: logger.error( "must send in either a remote db connection string or a memory database object" ) logger.debug("Initializing Memory Database") self.memDB.initEditValues(series_id) logger.debug("Finished Initializing Memory Database") self._populate_series() self.reset_filter()
def refreshTableSeries(self, db): """ Refreshes the objectlistview to include newly saved database series and preserve which series was 'checked' for plotting/editing :return: """ self.memDB = MemoryDatabase(db) object = self.dbservice.get_all_series() #checkedObjs = self.tblSeries.GetCheckedObjects() idList = [x.id for x in self.tblSeries.modelObjects] for x in object: if x.id not in idList: self.tblSeries.AddObject(x)
def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = test_util.add_series(self.session) self.memory_db.initEditValues(self.series.id) sorted_df = sorted(self.memory_db.df['LocalDateTime']) self.sdate = sorted_df[0]
def setup(self): self.app = wx.App() self.frame = wx.Frame(None) self.pltProb = plotProbability.plotProb(id=wxID_PAGEPROB, name='pltProb', parent=self.frame, pos=wx.Point(0, 0), size=wx.Size(605, 458), style=wx.TAB_TRAVERSAL) self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService( connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = add_series(self.session) print "Series: ", self.series self.memory_db.initEditValues(self.series.id)
class TestWizSave: def setup(self): conn_dict = {'engine': 'sqlite', 'address': ':memory:'} self.sm = ServiceManager(conn_dict=conn_dict) # set up remote Database # self.connection_string = "sqlite:///:memory:" self.series_service = self.sm.get_series_service() # SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.dvs_size = 100 self.series = test_util.add_series_bulk_data(self.session, dvs_size=self.dvs_size) assert self.series assert len(self.series.data_values) == self.dvs_size self.memory_database = MemoryDatabase() self.memory_database.set_series_service(self.series_service) # self.memory_database.initEditValues(self.series.id) self.app = wx.App() self.frame = wx.Frame(None) self.wizard = wizSave(self.frame,self.sm, self.sm.get_edit_service(self.series.id, self.memory_database))
def initTableSeries(self): """Set up columns and objects to be used in the objectlistview to be visible in the series selector :return: """ try: #self.dbservice = self.parent.Parent.createService() self.memDB = MemoryDatabase(self.dbservice) object = self.dbservice.get_all_series() if object: self.tblSeries.SetObjects(object) else: self.tblSeries.SetObjects(None) except AttributeError as e: logger.error(e)
def setup(self): self.app = wx.App() self.frame = wx.Frame(None) self.pltProb = plotProbability.plotProb(id=wxID_PAGEPROB, name='pltProb', parent=self.frame, pos=wx.Point(0, 0), size=wx.Size(605, 458), style=wx.TAB_TRAVERSAL) self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = add_series(self.session) print "Series: ", self.series self.memory_db.initEditValues(self.series.id)
def setup(self): #set up remote Database self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.dvs_size = 100 self.series = test_util.add_series_bulk_data(self.session, dvs_size=self.dvs_size) assert self.series assert len(self.series.data_values) == self.dvs_size self.memory_database = MemoryDatabase() self.memory_database.set_series_service(self.series_service) self.memory_database.initEditValues(self.series.id) self.app = wx.App() self.frame = wx.Frame(None) self.dataTable = FrmDataTable(self.frame)
class FrmSeriesSelector(clsSeriesSelector.ClsSeriesSelector): def __init__(self, *args, **kwargs): clsSeriesSelector.ClsSeriesSelector.__init__(self, *args, **kwargs) def initPubSub(self): #Publisher.subscribe(self.onEditButton, ("selectEdit")) Publisher.subscribe(self.refreshSeries, "refreshSeries") def resetDB(self, dbservice): """ :param dbservice: :return: """ if not self.rbAll.GetValue(): #self.GetEventHandler().ProcessEvent(wx.PyCommandEvent(wx.EVT_RADIOBUTTON.typeId, self.rbAll.Id)) wx.PostEvent(self.GetEventHandler(), wx.PyCommandEvent(wx.EVT_RADIOBUTTON.typeId, self.rbAll.Id)) self.rbAll.SetValue(True) #####INIT DB Connection self.dbservice = dbservice #self.refreshSeries() self.cbVariables.Clear() self.cbSites.Clear() self.siteList = None self.varList = None self.initTableSeries() self.initSVBoxes() self.Layout() def initTableSeries(self): """Set up columns and objects to be used in the objectlistview to be visible in the series selector :return: """ try: #self.dbservice = self.parent.Parent.createService() self.memDB = MemoryDatabase(self.dbservice) object = self.dbservice.get_all_series() if object: self.tblSeries.SetObjects(object) else: self.tblSeries.SetObjects(None) except AttributeError as e: logger.error(e) #self.tblSeries.SaveObject(object) def refreshTableSeries(self, db): """ Refreshes the objectlistview to include newly saved database series and preserve which series was 'checked' for plotting/editing :return: """ self.memDB = MemoryDatabase(db) object = self.dbservice.get_all_series() #checkedObjs = self.tblSeries.GetCheckedObjects() idList = [x.id for x in self.tblSeries.modelObjects] for x in object: if x.id not in idList: self.tblSeries.AddObject(x) #for x in checkedObjs: # super(FastObjectListView, self.tblSeries).SetCheckState(x, True) def refreshSeries(self): """ :return: """ self.dbservice = None self.dbservice = self.parent.Parent.createService() #self.refreshTableSeries(self.dbservice) self.resetDB(self.dbservice) logger.debug("Refresh Occurred") def initSVBoxes(self): """ :return: """ self.site_code = None self.variable_code = None #####INIT drop down boxes for Simple Filter try: self.siteList = self.dbservice.get_all_used_sites() for site in self.siteList: self.cbSites.Append(site.code + '-' + site.name) self.cbSites.SetSelection(0) self.site_code = self.siteList[0].code self.varList = self.dbservice.get_all_used_variables() for var in self.varList: self.cbVariables.Append(var.code + '-' + var.name) self.cbVariables.SetSelection(0) except AttributeError as e: logger.error(e) def OnTableRightDown(self, event): """Right click down menu :param event: :return: """ # build pop-up menu for right-click display self.selectedIndex = event.m_itemIndex #self.selectedID = self.tableSeries.getColumnText(event.m_itemIndex, 1) self.selectedID = self.tblSeries.GetSelectedObject().id # print self.selectedID popup_edit_series = wx.NewId() popup_plot_series = wx.NewId() popup_export_data = wx.NewId() popup_series_refresh = wx.NewId() popup_clear_selected = wx.NewId() popup_export_metadata = wx.NewId() popup_select_all = wx.NewId() popup_select_none = wx.NewId() popup_menu = wx.Menu() plotItem = popup_menu.Append(popup_plot_series, 'Plot') editItem = popup_menu.Append(popup_edit_series, 'Edit') self.Bind(wx.EVT_MENU, self.onRightPlot, plotItem) # TODO @jmeline needs to fix edit, it doesn't unedit when a plot is being edited self.Bind(wx.EVT_MENU, self.onRightEdit, editItem) # TODO @jmeline will refresh and clear selected as an enhancement #self.Bind(wx.EVT_MENU, self.onRightRefresh, popup_menu.Append(popup_series_refresh, 'Refresh')) #self.Bind(wx.EVT_MENU, self.onRightClearSelected, popup_menu.Append(popup_series_refresh, 'Clear Selected')) popup_menu.AppendSeparator() self.Bind(wx.EVT_MENU, self.onRightExData, popup_menu.Append(popup_export_data, 'Export Data')) self.Bind(wx.EVT_MENU, self.onRightExMeta, popup_menu.Append(popup_export_metadata, 'Export MetaData')) if self.isEditing: popup_menu.Enable(popup_edit_series, False) self.tblSeries.PopupMenu(popup_menu) event.Skip() def onPaneChanged(self, event=None): #if event: # print 'wx.EVT_COLLAPSIBLEPANE_CHANGED: %s\n' % event.Collapsed self.Layout() def onRbAdvancedRadiobutton(self, event): """ :param event: :return: """ self.cpnlSimple.Collapse(True) self.Layout() series_filter = frmQueryBuilder.frmQueryBuilder(self) self.filterlist = series_filter.ShowModal() event.Skip() def onRbAllRadiobutton(self, event): """ :param event: :return: """ if self.checkSite.GetValue() or self.checkVariable.GetValue(): self.checkSite.SetValue(False) self.checkVariable.SetValue(False) #logger.debug("onRbAllRadioButton called! ") self.cpnlSimple.Collapse(True) self.Layout() self.setFilter() event.Skip() def onRbSimpleRadiobutton(self, event): """ :param event: :return: """ self.cpnlSimple.Expand() self.Layout() ## if not self.checkSite.GetValue() and not self.checkVariable.GetValue(): self.setFilter() return self.setFilter(self.site_code, self.variable_code) event.Skip() def onRightPlot(self, event): """ :param event: :return: """ object = self.tblSeries.GetSelectedObject() self.tblSeries.ToggleCheck(object) self.onReadyToPlot(event) event.Skip() def onRightEdit(self, event): """ :param event: :return: """ Publisher.sendMessage(("selectEdit"), event=event) if self.isEditing: Publisher.sendMessage("toggleEdit", checked=True) event.Skip() # allows user to right-click refresh the Series Selector def onRightRefresh(self, event): """ :param event: :return: """ self.refreshSeries() event.Skip() def onRightClearSelected(self, event): """ :param event: :return: """ event.Skip() def onRightExData(self, event): """ :param event: :return: """ dlg = wx.FileDialog(self, "Choose a save location", '', "", "*.csv", wx.SAVE | wx.OVERWRITE_PROMPT) if dlg.ShowModal() == wx.ID_OK: full_path = os.path.join(dlg.GetDirectory(), dlg.GetFilename()) #series_id = self.tableSeries.getColumnText(self.selectedIndex, 1) series_id = self.tblSeries.GetSelectedObject().id self.export_service.export_series_data(series_id, full_path, True, True, True, True, True, True, True) self.Close() dlg.Destroy() event.Skip() def onRightExMeta(self, event): """ :param event: :return: """ dlg = wx.FileDialog(self, "Choose a save location", '', "", "*.xml", wx.SAVE | wx.OVERWRITE_PROMPT) if dlg.ShowModal() == wx.ID_OK: full_path = os.path.join(dlg.GetDirectory(), dlg.GetFilename()) self.selectedIndex = self.tblSeries.GetSelectedObject().id #series_id = self.tableSeries.getColumnText(self.selectedIndex, 1) #print "series_id", series_id self.export_service.export_series_metadata(self.selectedIndex, full_path) self.Close() dlg.Destroy() event.Skip() def onCbSitesCombobox(self, event): """ :param event: :return: """ if self.checkSite.GetValue(): self.site_code = self.siteList[event.GetSelection()].code self.varList = self.dbservice.get_variables_by_site_code(self.site_code) self.cbVariables.Clear() for var in self.varList: self.cbVariables.Append(var.code + '-' + var.name) self.cbVariables.SetSelection(0) if (self.checkSite.GetValue() and not self.checkVariable.GetValue()): self.variable_code = None self.setFilter(site_code=self.site_code, var_code=self.variable_code) event.Skip() def onCbVariablesCombobox(self, event): """ :param event: :return: """ if self.checkVariable.GetValue(): self.variable_code = self.varList[event.GetSelection()].code if (not self.checkSite.GetValue() and self.checkVariable.GetValue()): self.site_code = None self.setFilter(site_code=self.site_code, var_code=self.variable_code) event.Skip() def siteAndVariables(self): """ :return: """ self.site_code = self.siteList[self.cbSites.Selection].code self.cbVariables.Clear() self.varList = self.dbservice.get_variables_by_site_code(self.site_code) for var in self.varList: self.cbVariables.Append(var.code + '-' + var.name) self.cbVariables.SetSelection(0) try: self.variable_code = self.varList[self.cbVariables.Selection].code self.setFilter(site_code=self.site_code, var_code=self.variable_code) self.cbVariables.Enabled = True self.cbSites.Enabled = True except IndexError: pass def siteOnly(self): """ :return: """ self.cbVariables.Enabled = False self.cbSites.Enabled = True self.variable_code = None self.site_code = self.siteList[self.cbSites.Selection].code self.setFilter(site_code=self.site_code) def variableOnly(self): """ :return: """ self.site_code = None self.cbVariables.Clear() self.varList = self.dbservice.get_all_used_variables() for var in self.varList: self.cbVariables.Append(var.code + '-' + var.name) self.cbVariables.SetSelection(0) self.cbSites.Enabled = False self.cbVariables.Enabled = True self.variable_code = self.varList[0].code self.setFilter(var_code=self.variable_code) def onCheck(self, event): """ :param event: :return: """ # self.tableSeries.DeleteAllItems() if self.rbAll.GetValue(): #logger.debug("Called!") self.rbAll.SetValue(False) self.rbSimple.SetValue(True) if not self.checkSite.GetValue() and not self.checkVariable.GetValue(): self.setFilter() self.cbSites.Enabled = False self.cbVariables.Enabled = False elif self.checkSite.GetValue(): if self.checkVariable.GetValue(): self.siteAndVariables() else: self.siteOnly() else: if self.checkVariable.GetValue(): self.variableOnly() else: self.cbSites.Enabled = False self.cbVariables.Enabled = False event.Skip() def setFilter(self, site_code='', var_code='', advfilter=''): """ :param site_code: :param var_code: :param advfilter: :return: """ if site_code and var_code: self.siteFilter = TextSearch(self.tblSeries, columns=self.tblSeries.columns[3:4],text=site_code) self.variableFilter = TextSearch(self.tblSeries, columns=self.tblSeries.columns[6:7],text=var_code) self.tblSeries.SetFilter(Chain(self.siteFilter, self.variableFilter)) elif site_code: self.tblSeries.SetFilter(TextSearch(self.tblSeries, columns=self.tblSeries.columns[3:4], text=site_code)) elif var_code: self.tblSeries.SetFilter(TextSearch(self.tblSeries, columns=self.tblSeries.columns[6:7], text=var_code)) elif advfilter: self.tblSeries.SetFilter(advfilter) else: self.tblSeries.SetFilter(TextSearch(self.tblSeries, columns=self.tblSeries.columns[0:1])) self.tblSeries.RepopulateList() def onReadyToPlot(self, event): """Plots a series selected from the series selector :param event: EVT_OVL_CHECK_EVENT type """ checkedCount = len(self.tblSeries.GetCheckedObjects()) Publisher.sendMessage("EnablePlotButtons", plot=0, isActive=(checkedCount > 0)) try: object = event.object except: object = self.tblSeries.GetSelectedObject() if not self.tblSeries.IsChecked(object): Publisher.sendMessage("removePlot", seriesID=object.id) else: #logger.debug("%d" % (len(self.tblSeries.GetCheckedObjects()))) self.parent.Parent.addPlot(self.memDB, object.id) Publisher.sendMessage("updateCursor", selectedObject=object) self.Refresh() def getSelectedObject(self, event): """Capture the currently selected Object to be used for editing :param event: wx.EVT_LIST_ITEM_FOCUSED type """ object = event.GetEventObject() editingObject = object.innerList[object.FocusedItem] self.tblSeries.currentlySelectedObject = editingObject ## update Cursor if self.parent.Parent.pnlPlot._seriesPlotInfo: if self.parent.Parent.pnlPlot._seriesPlotInfo.isPlotted(editingObject.id): #print "Updating Cursor", editingObject.id Publisher.sendMessage("updateCursor", selectedObject=editingObject) def onReadyToEdit(self): """Choose a series to edit from the series selector :return: """ ovl = self.tblSeries object = ovl.currentlySelectedObject if object is None: # # Select the first one if len(ovl.modelObjects) == 0: logger.fatal("There are no model objects available to edit") raise Exception() object = ovl.modelObjects[0] if len(ovl.GetCheckedObjects()) <= ovl.allowedLimit: if object not in ovl.GetCheckedObjects(): ovl.ToggleCheck(object) self.memDB.initEditValues(object.id) self.isEditing = True ovl.editingObject = object ovl.RefreshObject(ovl.editingObject) return True, object.id, self.memDB else: isSelected = False logger.debug("series was not checked") val_2 = wx.MessageBox("Visualization is limited to 6 series.", "Can't add plot", wx.OK | wx.ICON_INFORMATION) self.isEditing = False ovl.editingObject = None return False, object.id, self.memDB def stopEdit(self): """When edit button is untoggled, the editing feature closes :return: """ self.isEditing = False self.tblSeries.RefreshObject(self.tblSeries.editingObject) self.tblSeries.editingObject = None self.memDB.stopEdit() def isEditing(self): """ :return: """ return self.isEditing def _rowFormatter(self, listItem, object): """Handles the formatting of rows for object list view :param: wx.ListCtrl listitem :param: ModelObject object :rtype: None """ '''
class EditService(): # Mutual exclusion: cursor, or connection_string def __init__(self, series_id, connection=None, connection_string="", debug=False): ''' :param series_id: :param connection: memory database, contains connection to remote database :param connection_string: connection to remote database :param debug: :return: ''' self._series_id = series_id self._filter_from_selection = False self._debug = debug if connection_string is "" and connection is not None: self.memDB= connection elif connection_string is not "" and connection is None: from odmtools.odmdata import MemoryDatabase self.memDB= MemoryDatabase() self.memDB.set_series_service(SeriesService(connection_string, False)) else: logger.error("must send in either a remote db connection string or a memory database object") logger.debug("Initializing Memory Database") self.memDB.initEditValues(series_id) logger.debug("Finished Initializing Memory Database") self._populate_series() self.reset_filter() def get_series_service(self): return self.memDB.series_service def _populate_series(self): # [(ID, value, datetime), ...] #self._cursor.execute("SELECT ValueID, DataValue, LocalDateTime FROM DataValues ORDER BY LocalDateTime") self._series_points_df = self.memDB.getDataValuesDF() def _test_filter_previous(self): ''' if not self._filter_from_selection: self.reset_filter() ''' df = None if not self._filter_from_selection: df = self._series_points_df else: df = self.filtered_dataframe # Ensure that we're not working with an empty dataframe if isinstance(df, pd.DataFrame): if df.empty: return self._series_points_df else: if not df: return self._series_points_df return df def datetime2dataframe(self, datetime_list): """ Converts datetime_list to a pandas Dataframe :param datetime_list: :return Pandas.DataFrame: """ result = None if isinstance(datetime_list, list): result = pd.DataFrame(datetime_list, columns=["LocalDateTime"]) result.set_index("LocalDateTime", inplace=True) return result ################### # Stubs ################### def selectPointsStub(self): """ :param filtered_dataframe: :return: """ ## Convert dataframe into list of datetimes filtered_dataframe = self.get_filtered_points() if isinstance(filtered_dataframe, pd.DataFrame): if not filtered_dataframe.empty: datetime_list = filtered_dataframe.index.to_pydatetime() return datetime_list.tolist() return [] ################### # Filters ################### # operator is a character, either '<' or '>' def filter_value(self, value, ops): df = self._test_filter_previous() if ops == '>': self.filtered_dataframe = df[df['DataValue'] > value] if ops == '<': self.filtered_dataframe = df[df['DataValue'] < value] def filter_date(self, before, after): df = self._test_filter_previous() if before and after: self.filtered_dataframe = df[(df.index < before) & (df.index > after)] def fill_gap(self, gap, fill): df = self.memDB.getDataValuesDF() gaps= self.find_gaps(df, gap[0], gap[1]) points = [] series= self.memDB.series timegap = np.timedelta64(fill[0], self.time_units[fill[1]]) #if gaps is not of type dataframe- put it in a dataframe #if not isinstance(gaps, pd.DataFrame for g in gaps.iterrows(): row = g[1] e = row.datetime s = row.dateprev #prime the loop s = s + timegap # for each gap time period in the larger gap ( until datetime = prev value) while s < e: utc_offset = (series.begin_date_time-series.begin_date_time_utc).total_seconds()/3600 points.append((-9999, None, s, utc_offset, s, None, None, u'nc', None, None, series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id)) #('-9999', None, DATE, series.begin_date_time_utc, UTCDATE, None, None, u'nc', None, None, # series.site_id, series.variable_id, series.method_id, series.source_id, # series.quality_control_level_id s = s + timegap #print points self.add_points(points) time_units = { 'second': 's', 'minute': 'm', 'hour': 'h', 'day': 'D', 'week': 'W', 'month': 'M', 'year': 'Y' } # Data Gaps def find_gaps(self, df, value, time_period): # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps copy_df = df copy_df['datetime'] = df.index copy_df['dateprev'] = copy_df['datetime'].shift() # ensure that 'value' is an integer if not isinstance(value, int): value = int(value) # create a bool column indicating which rows meet condition filtered_results = copy_df['datetime'].diff() > np.timedelta64(value, self.time_units[time_period]) # filter on rows that passed previous condition return copy_df[filtered_results] def data_gaps(self, value, time_period): df = self._test_filter_previous() copy_df = self.find_gaps(df, value, time_period) print (copy_df) # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner') # clean up del copy_df self.filtered_dataframe= df[df.index.isin(newdf.drop_duplicates().dropna())] def change_value_threshold(self, value, operator): df = self._test_filter_previous() # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps copy_df = df copy_df['values'] = df['DataValue'] copy_df['diff'] = copy_df['values'].shift() copy_df["diff_date"] = copy_df['LocalDateTime'].shift() copy_df['change_threshold'] = abs(df['values'] - df['diff']) if not isinstance(value, float): logger.error("Need to have a float") return copy_df['threshold'] = value if operator == ">": copy_df['matches'] = df['change_threshold'] >= copy_df['threshold'] if operator == "<": copy_df['matches'] = df['change_threshold'] <= copy_df['threshold'] filtered_df = copy_df[copy_df['matches']] tmplist = filtered_df['diff_date'].tolist() + filtered_df.index.tolist() del copy_df self.filtered_dataframe = df[df.index.isin(tmplist)] #Duplicate values filter def duplicate_value_filter(self): df = self._test_filter_previous() #self.filtered_dataframe= df[df.index.get_duplicates()] self.filtered_dataframe= df[df.index.isin(df.index.get_duplicates())] #self.filtered_dataframe = df[df['DataValue'] < value] def select_points_tf(self, tf_list): self._filter_list = tf_list #def select_points(self, id_list=[], datetime_list=[]): def select_points(self, id_list=[], dataframe=[]): #self.reset_filter() # This should be either one or the other. If it's both, id is used first. # If neither are set this function does nothing. if len(id_list) > 0: for i in range(len(self._series_points)): if self._series_points[i][0] in id_list: self._filter_list[i] = True if isinstance(dataframe, pd.DataFrame): result = dataframe.index.astype(datetime.datetime) self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(dataframe.index)] def reset_filter(self): self.filtered_dataframe = None def filter_from_previous(self, value): self._filter_from_selection = value def get_toggle(self): return self._filter_from_selection ################### # Gets ################### def get_series(self): return self.memDB.series_service.get_series_by_id(self._series_id) def get_series_points(self): # all point in the series return self._series_points def get_series_points_df(self): """ :return Pandas DataFrame: """ return self._series_points_df def get_filtered_points(self): """ :return Pandas DataFrame: """ if isinstance(self.filtered_dataframe, pd.DataFrame): if self.filtered_dataframe.empty: return None else: if not self.filtered_dataframe: return None if len(self.filtered_dataframe) > 0: return self.filtered_dataframe return None def get_filtered_dates(self): return self.filtered_dataframe def get_filter_list(self): # true or false list the length of the entire series. true indicate the point is selected return self._filter_list def get_qcl(self, qcl_id): return self.memDB.series_service.get_qcl_by_id(qcl_id) def get_method(self, method_id): return self.memDB.series_service.get_method_by_id(method_id) def get_variable(self, variable_id): logger.debug(variable_id) return self.memDB.series_service.get_variable_by_id(variable_id) ################# # Edits ################# def change_value(self, value, operator): filtered_points = self.get_filtered_points() ids = filtered_points.index.tolist() self.memDB.updateValue(ids, operator, float(value)) self._populate_series() ## update filtered_dataframe self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] def add_points(self, points): # todo: add the ability to send in multiple datetimes to a single 'point' self.memDB.addPoints(points) self._populate_series() self.reset_filter() def delete_points(self): filtered_points = self.get_filtered_points() if not filtered_points.empty: values = filtered_points.index.tolist() self.memDB.delete(values) self._populate_series() self.filtered_dataframe = None def interpolate(self): ''' In [75]: ser = Series(np.sort(np.random.uniform(size=100))) # interpolate at new_index In [76]: new_index = ser.index | Index([49.25, 49.5, 49.75, 50.25, 50.5, 50.75]) In [77]: interp_s = ser.reindex(new_index).interpolate(method='pchip') ''' tmp_filter_list =self.get_filtered_points() df = self._series_points_df issel = df.index.isin(tmp_filter_list.index) mdf = df["DataValue"].mask(issel) mdf.interpolate(method = "time", inplace=True) tmp_filter_list["DataValue"]=mdf[issel] ids = tmp_filter_list.index.tolist() #update_list = [(row["DataValue"], row["ValueID"]) for index, row in tmp_filter_list.iterrows()] update_list = [{"value": row["DataValue"], "id": index} for index, row in tmp_filter_list.iterrows()] self.memDB.update(update_list) self._populate_series() self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] def drift_correction(self, gap_width): if self.isOneGroup(): tmp_filter_list =self.get_filtered_points() startdate =tmp_filter_list.index[0] x_l = (tmp_filter_list.index[-1]-startdate).total_seconds() #nodv= self.memDB.series_service.get_variable_by_id(self.memDB.df["VariableID"][0]) nodv = self.memDB.series.variable.no_data_value # y_n = y_0 + G(x_i / x_l) f = lambda row : row["DataValue"]+(gap_width * ((row.name-startdate).total_seconds() / x_l)) if row["DataValue"] != nodv else row["DataValue"] tmp_filter_list["DataValue"]=tmp_filter_list.apply(f, axis = 1) update_list = [{"value": row["DataValue"], "id":index} for index, row in tmp_filter_list.iterrows()] ids = tmp_filter_list.index.tolist() self.memDB.update(update_list) self._populate_series() self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] return True return False def isOneGroup(self): issel = self._series_points_df.index.isin(self.get_filtered_points().index) found_group = False count = 0 for x in issel: if x: if not found_group: found_group=True count =count+1 else: found_group = False if count >1: return False if count == 1: return True def flag(self, qualifier_id): filtered_points = self.get_filtered_points() ''' query = "UPDATE DataValues SET QualifierID = %s WHERE ValueID = ?" % (qualifier_id) #self._cursor.executemany(query, [(str(x[0]),) for x in filtered_points]) self._cursor.executemany(query, [(str(x),) for x in filtered_points["ValueID"].astype(int).tolist()]) ''' self.memDB.updateFlag(filtered_points.index.tolist(), qualifier_id) def updateValues(self, values): """ :param values: pandas Dataframe - must contain a "datavalues" column and a date time as the index :return: """ if values is None: print("please send in a valid DataFrame object") return update_list = [{"value": row["DataValue"], "id": index} for index, row in values.iterrows()] ids = values.index.tolist() self.memDB.update(update_list) self._populate_series() self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] ################### # Save/Restore ################### def restore(self): self.memDB.rollback() self._populate_series() self.reset_filter() def updateSeries(self, var=None, method=None, qcl=None, is_new_series=False, overwrite = True, append = False): """ :param var: :param method: :param qcl: :param is_new_series: :return: """ var_id = var.id if var is not None else None method_id = method.id if method is not None else None qcl_id = qcl.id if qcl is not None else None #self.memDB.changeSeriesIDs(var_id, method_id, qcl_id) dvs = self.memDB.getDataValuesDF() if var_id is not None: dvs["VariableID"] = var_id if method_id is not None: dvs["MethodID"] = method_id if qcl_id is not None: dvs["QualityControlLevelID"] = qcl_id #if is new series remove valueids #if is_new_series: dvs["ValueID"] = None ''' for dv in dvs: dv.id = None ''' series = self.memDB.series_service.get_series_by_id(self._series_id) logger.debug("original editing series id: %s" % str(series.id)) if (var or method or qcl ): tseries = self.memDB.series_service.get_series_by_id_quint(site_id=int(series.site_id), var_id=var_id if var else int(series.variable_id), method_id=method_id if method else int( series.method_id), source_id=series.source_id, qcl_id=qcl_id if qcl else int( series.quality_control_level_id)) if tseries: logger.debug("Save existing series ID: %s" % str(tseries.id)) series = tseries else: print "Series doesn't exist (if you are not, you should be running SaveAs)" if is_new_series: series = series_module.copy_series(series) if var: series.variable_id = var_id series.variable_code = var.code series.variable_name = var.name series.speciation = var.speciation series.variable_units_id = var.variable_unit_id series.variable_units_name = var.variable_unit.name series.sample_medium = var.sample_medium series.value_type = var.value_type series.time_support = var.time_support series.time_units_id = var.time_unit_id series.time_units_name = var.time_unit.name series.data_type = var.data_type series.general_category = var.general_category if method: series.method_id = method_id series.method_description = method.description if qcl: series.quality_control_level_id = qcl_id series.quality_control_level_code = qcl.code ''' dvs["LocalDateTime"] = pd.to_datetime(dvs["LocalDateTime"]) dvs["DateTimeUTC"] = pd.to_datetime(dvs["DateTimeUTC"]) ''' form = "%Y-%m-%d %H:%M:%S" if not append: series.begin_date_time = datetime.datetime.strptime(str(np.min(dvs["LocalDateTime"])), form)#np.min(dvs["LocalDateTime"])#dvs[c0].local_date_time series.end_date_time = datetime.datetime.strptime(str(np.max(dvs["LocalDateTime"])), form)#np.max(dvs["LocalDateTime"])#dvs[-1].local_date_time series.begin_date_time_utc = datetime.datetime.strptime(str(np.min(dvs["DateTimeUTC"])), form) #dvs[0].date_time_utc series.end_date_time_utc = datetime.datetime.strptime(str(np.max(dvs["DateTimeUTC"])), form) #dvs[-1].date_time_utc series.value_count = len(dvs) ## Override previous save if not is_new_series: # delete old dvs #pass self.memDB.series_service.delete_values_by_series(series) elif append: #if series end date is after dvs startdate dbend = series.end_date_time dfstart = datetime.datetime.strptime(str(np.min(dvs["LocalDateTime"])), form) overlap = dbend>= dfstart #leave series start dates to those previously set series.end_date_time = datetime.datetime.strptime(str(np.max(dvs["LocalDateTime"])), form) series.end_date_time_utc = datetime.datetime.strptime(str(np.max(dvs["DateTimeUTC"])), form) #TODO figure out how to calculate the new value count series.value_count = len(dvs) if overlap: if overwrite: #remove values from the database self.memDB.series_service.delete_values_by_series(series, startdate=dfstart) else: #remove values from df dvs = dvs[dvs["LocalDateTime"] > dbend] #logger.debug("series.data_values: %s" % ([x for x in series.data_values])) dvs.drop('ValueID', axis=1, inplace=True) return series, dvs def save(self): """ Save to an existing catalog :param var: :param method: :param qcl: :return: """ series, dvs = self.updateSeries(is_new_series=False) if self.memDB.series_service.save_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Save was unsuccessful") return False def save_as(self, var=None, method=None, qcl=None): """ :param var: :param method: :param qcl: :return: """ series, dvs = self.updateSeries(var, method, qcl, is_new_series=True) if self.memDB.series_service.save_new_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Save As Function was Unsuccessful") return False def save_appending(self, var= None, method = None, qcl=None, overwrite=False): series, dvs = self.updateSeries(var, method, qcl, is_new_series=False, append= True, overwrite=overwrite) if self.memDB.series_service.save_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Append Existing Function was Unsuccessful") return False def save_existing(self, var=None, method=None, qcl=None): """ :param var: :param method: :param qcl: :return: """ series, dvs = self.updateSeries(var, method, qcl, is_new_series=False) if self.memDB.series_service.save_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Save As Existing Function was Unsuccessful") return False def create_qcl(self, code, definition, explanation): return self.memDB.series_service.create_qcl(code, definition, explanation) def create_method(self, description, link): return self.memDB.series_service.create_method(description, link) def create_qualifier(self, code, definition): return self.memDB.series_service.create_qualifier(code, definition) def create_variable(self, code, name, speciation, variable_unit_id, sample_medium, value_type, is_regular, time_support, time_unit_id, data_type, general_category, no_data_value): return self.memDB.series_service.create_variable(code, name, speciation, variable_unit_id, sample_medium, value_type, is_regular, time_support, time_unit_id, data_type, general_category, no_data_value) def reconcile_dates(self, parent_series_id): # FUTURE FEATURE: pull in new field data from another series and add to this series # (i.e one series contains new field data of an edited series at a higher qcl) pass
class TestMemoryDB: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = test_util.add_series(self.session) self.memory_db.initEditValues(self.series.id) def test_get_data_values(self): dvs = self.memory_db.getDataValuesDF() assert len(dvs) == 10 def test_update_points(self): self.memory_db.update([{"value": 15, "id": 1}]) dvs = self.memory_db.getDataValuesDF() print dvs["DataValue"] assert dvs["DataValue"][1 - 1] == 9 def test_update_value(self): self.memory_db.updateValue([1], "+", 5) dvs = self.memory_db.getDataValuesDF() assert dvs["DataValue"][1 - 1] == 9 def test_add_points(self): # with pytest.raises(NotImplementedError): point = [ ( "-9999", None, datetime.datetime(2011, 3, 25, 0, 0), "-7", datetime.datetime(2015, 3, 25, 7, 0), None, None, u"nc", None, None, self.series.site_id, self.series.variable_id, self.series.method_id, self.series.source_id, self.series.quality_control_level_id, ) ] self.memory_db.addPoints(point) dvs = self.memory_db.getDataValuesDF() assert len(dvs) == 11 assert dvs["DataValue"][-1] == -9999 def test_update_flag(self): self.memory_db.updateFlag([5], "50") dvs = self.memory_db.getDataValuesDF() assert dvs["QualifierID"][5 - 1] == "50" def test_delete_points(self): stlen = len(self.memory_db.getDataValuesDF()) self.memory_db.delete([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) dvs = self.memory_db.getDataValuesDF() assert len(dvs) == stlen - 10
class TestMemoryDB: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService( connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = test_util.add_series(self.session) self.memory_db.initEditValues(self.series.id) def test_get_data_values(self): dvs = self.memory_db.getDataValuesDF() assert len(dvs) == 10 def test_update_points(self): self.memory_db.update([{"value": 15, "id": 1}]) dvs = self.memory_db.getDataValuesDF() print dvs["DataValue"] assert dvs["DataValue"][1 - 1] == 9 def test_update_value(self): self.memory_db.updateValue([1], '+', 5) dvs = self.memory_db.getDataValuesDF() assert dvs["DataValue"][1 - 1] == 9 def test_add_points(self): #with pytest.raises(NotImplementedError): point = [('-9999', None, datetime.datetime(2011, 3, 25, 0, 0), '-7', datetime.datetime(2015, 3, 25, 7, 0), None, None, u'nc', None, None, self.series.site_id, self.series.variable_id, self.series.method_id, self.series.source_id, self.series.quality_control_level_id)] self.memory_db.addPoints(point) dvs = self.memory_db.getDataValuesDF() assert len(dvs) == 11 assert dvs["DataValue"][-1] == -9999 def test_update_flag(self): self.memory_db.updateFlag([5], '50') dvs = self.memory_db.getDataValuesDF() assert dvs["QualifierID"][5 - 1] == '50' def test_delete_points(self): stlen = len(self.memory_db.getDataValuesDF()) self.memory_db.delete([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) dvs = self.memory_db.getDataValuesDF() assert len(dvs) == stlen - 10
def runODM(): app = wx.App(False) #frame = create(None) #frame.Show() app.MainLoop() if __name__ == '__main__': logger.debug("Welcome to ODMTools Python. Please wait as system loads") # https://docs.python.org/2/library/multiprocessing.html#miscellaneous # Add support for when a program which uses multiprocessing has been frozen to produce a Windows executable. # (Has been tested with py2exe, PyInstaller and cx_Freeze.) # One needs to call this function straight after the if __name__ == '__main__' line of the main module. # If the freeze_support() line is omitted then trying to run the frozen executable will raise RuntimeError. # If the module is being run normally by the Python interpreter then freeze_support() has no effect. freeze_support() # Determine the number of CPU's available numproc = cpu_count() # Initialize TaskServer. # This class starts the processes before starting wxpython and is needed tsmp = TaskServerMP(numproc=numproc) memdb = MemoryDatabase() # Build app with taskserver included app = MyApp(False, taskserver=tsmp, memdb=memdb) app.MainLoop()
class TestMemoryDB: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = test_util.add_series(self.session) self.memory_db.initEditValues(self.series.id) sorted_df = sorted(self.memory_db.df['LocalDateTime']) self.sdate = sorted_df[0] def test_get_data_values(self): dvs = self.memory_db.getDataValuesDF() assert len(dvs) == 10 def test_build_series(self): dvs = 100 self.series = test_util.add_series_bulk_data(self.session, dvs_size=dvs) assert self.series assert len(self.series.data_values) == dvs def test_update_points(self): self.memory_db.update([{"value":15,"id":self.sdate}]) dvs = self.memory_db.getDataValuesDF() print dvs["DataValue"] assert dvs["DataValue"][0] == 15 def test_update_value(self): self.memory_db.updateValue([self.sdate],'+', 5 ) dvs = self.memory_db.getDataValuesDF() assert dvs["DataValue"][0] == 14 def test_add_points(self): #with pytest.raises(NotImplementedError): assert len(self.memory_db.getDataValuesDF().index)==10 point = [('-9999', None, datetime.datetime(2011, 3, 25, 0, 0), '-7', datetime.datetime(2015, 3, 25, 7, 0), None, None, u'nc', None, None, self.series.site_id, self.series.variable_id, self.series.method_id, self.series.source_id, self.series.quality_control_level_id)] self.memory_db.addPoints(point) dvs = self.memory_db.getDataValuesDF() assert len(dvs.index) == 11 assert dvs["DataValue"][0] == -9999 def test_update_flag(self): self.memory_db.updateFlag([self.sdate], '50') dvs=self.memory_db.getDataValuesDF() assert dvs["QualifierID"][0] == '50' def test_delete_points(self): stlen= len(self.memory_db.df.index) self.memory_db.delete(self.memory_db.df["LocalDateTime"].tolist()[0:10]) dvs = self.memory_db.getDataValuesDF() assert len(dvs.index) == stlen-10
class EditService(): # Mutual exclusion: cursor, or connection_string def __init__(self, series_id, connection=None, connection_string="", debug=False): ''' :param series_id: :param connection: memory database, contains connection to remote database :param connection_string: connection to remote database :param debug: :return: ''' self._series_id = series_id self._filter_from_selection = False self._debug = debug if connection_string is "" and connection is not None: self.memDB = connection #self._series_service = self.memDB.series_service#SeriesService(connection_string, debug) elif connection_string is not "" and connection is None: from odmtools.odmdata import MemoryDatabase self.memDB = MemoryDatabase() #(series_service) self.memDB.set_series_service( SeriesService(connection_string, False)) else: logger.error( "must send in either a remote db connection string or a memory database object" ) logger.debug("Initializing Memory Database") self.memDB.initEditValues(series_id) logger.debug("Finished Initializing Memory Database") self._populate_series() self.reset_filter() def get_series_service(self): return self.memDB.series_service def _populate_series(self): # [(ID, value, datetime), ...] #self._cursor.execute("SELECT ValueID, DataValue, LocalDateTime FROM DataValues ORDER BY LocalDateTime") self._series_points_df = self.memDB.getDataValuesDF() def _test_filter_previous(self): ''' if not self._filter_from_selection: self.reset_filter() ''' df = None if not self._filter_from_selection: df = self._series_points_df else: df = self.filtered_dataframe # Ensure that we're not working with an empty dataframe if isinstance(df, pd.DataFrame): if df.empty: return self._series_points_df else: if not df: return self._series_points_df return df def datetime2dataframe(self, datetime_list): """ Converts datetime_list to a pandas Dataframe :param datetime_list: :return Pandas.DataFrame: """ result = None if isinstance(datetime_list, list): result = pd.DataFrame(datetime_list, columns=["LocalDateTime"]) result.set_index("LocalDateTime", inplace=True) return result ################### # Stubs ################### def selectPointsStub(self): """ :param filtered_dataframe: :return: """ ## Convert dataframe into list of datetimes filtered_dataframe = self.get_filtered_points() if isinstance(filtered_dataframe, pd.DataFrame): if not filtered_dataframe.empty: datetime_list = filtered_dataframe.index.to_pydatetime() return datetime_list.tolist() return [] ################### # Filters ################### # operator is a character, either '<' or '>' def filter_value(self, value, ops): df = self._test_filter_previous() if ops == '>': self.filtered_dataframe = df[df['DataValue'] > value] if ops == '<': self.filtered_dataframe = df[df['DataValue'] < value] def filter_date(self, before, after): df = self._test_filter_previous() if before and after: self.filtered_dataframe = df[(df.index < before) & (df.index > after)] # Data Gaps def data_gaps(self, value, time_period): df = self._test_filter_previous() time_units = { 'second': 's', 'minute': 'm', 'hour': 'h', 'day': 'D', 'week': 'W', 'month': 'M', 'year': 'Y' } # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps copy_df = df copy_df['datetime'] = df.index copy_df['dateprev'] = copy_df['datetime'].shift() # ensure that 'value' is an integer if not isinstance(value, int): value = int(value) # create a bool column indicating which rows meet condition filtered_results = copy_df['datetime'].diff() >= np.timedelta64( value, time_units[time_period]) # filter on rows that passed previous condition copy_df = copy_df[filtered_results] # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner') self.filtered_dataframe = df[df.index.isin( newdf.drop_duplicates().dropna())] # clean up del copy_df del filtered_results del newdf def change_value_threshold(self, value, operator): df = self._test_filter_previous() # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps copy_df = df copy_df['values'] = df['DataValue'] copy_df['diff'] = copy_df['values'].shift() copy_df["diff_date"] = copy_df['LocalDateTime'].shift() copy_df['change_threshold'] = abs(df['values'] - df['diff']) if not isinstance(value, float): logger.error("Need to have a float") return copy_df['threshold'] = value if operator == ">": copy_df['matches'] = df['change_threshold'] >= copy_df['threshold'] if operator == "<": copy_df['matches'] = df['change_threshold'] <= copy_df['threshold'] filtered_df = copy_df[copy_df['matches']] tmplist = filtered_df['diff_date'].tolist() + filtered_df.index.tolist( ) del copy_df self.filtered_dataframe = df[df.index.isin(tmplist)] def select_points_tf(self, tf_list): self._filter_list = tf_list #def select_points(self, id_list=[], datetime_list=[]): def select_points(self, id_list=[], dataframe=[]): #self.reset_filter() # This should be either one or the other. If it's both, id is used first. # If neither are set this function does nothing. if len(id_list) > 0: for i in range(len(self._series_points)): if self._series_points[i][0] in id_list: self._filter_list[i] = True if isinstance(dataframe, pd.DataFrame): result = dataframe.index.astype(datetime.datetime) self.filtered_dataframe = self._series_points_df[ self._series_points_df.index.isin(dataframe.index)] def reset_filter(self): self.filtered_dataframe = None def filter_from_previous(self, value): self._filter_from_selection = value def get_toggle(self): return self._filter_from_selection ################### # Gets ################### def get_series(self): return self.memDB.series_service.get_series_by_id(self._series_id) def get_series_points(self): # all point in the series return self._series_points def get_series_points_df(self): """ :return Pandas DataFrame: """ return self._series_points_df def get_filtered_points(self): """ :return Pandas DataFrame: """ if isinstance(self.filtered_dataframe, pd.DataFrame): if self.filtered_dataframe.empty: return None else: if not self.filtered_dataframe: return None if len(self.filtered_dataframe) > 0: return self.filtered_dataframe return None def get_filtered_dates(self): return self.filtered_dataframe def get_filter_list(self): # true or false list the length of the entire series. true indicate the point is selected return self._filter_list def get_qcl(self, qcl_id): return self.memDB.series_service.get_qcl_by_id(qcl_id) def get_method(self, method_id): return self.memDB.series_service.get_method_by_id(method_id) def get_variable(self, variable_id): logger.debug(variable_id) return self.memDB.series_service.get_variable_by_id(variable_id) ################# # Edits ################# def change_value(self, value, operator): filtered_points = self.get_filtered_points() ids = filtered_points.index.tolist() self.memDB.updateValue(ids, operator, float(value)) self._populate_series() ## update filtered_dataframe self.filtered_dataframe = self._series_points_df[ self._series_points_df.index.isin(ids)] def add_points(self, points): # todo: add the ability to send in multiple datetimes to a single 'point' self.memDB.addPoints(points) self._populate_series() self.reset_filter() def delete_points(self): filtered_points = self.get_filtered_points() if not filtered_points.empty: values = filtered_points.index.tolist() self.memDB.delete(values) self._populate_series() self.filtered_dataframe = None def interpolate(self): ''' In [75]: ser = Series(np.sort(np.random.uniform(size=100))) # interpolate at new_index In [76]: new_index = ser.index | Index([49.25, 49.5, 49.75, 50.25, 50.5, 50.75]) In [77]: interp_s = ser.reindex(new_index).interpolate(method='pchip') ''' tmp_filter_list = self.get_filtered_points() df = self._series_points_df issel = df.index.isin(tmp_filter_list.index) mdf = df["DataValue"].mask(issel) mdf.interpolate(method="time", inplace=True) tmp_filter_list["DataValue"] = mdf[issel] ids = tmp_filter_list.index.tolist() #update_list = [(row["DataValue"], row["ValueID"]) for index, row in tmp_filter_list.iterrows()] update_list = [{ "value": row["DataValue"], "id": index } for index, row in tmp_filter_list.iterrows()] self.memDB.update(update_list) self._populate_series() self.filtered_dataframe = self._series_points_df[ self._series_points_df.index.isin(ids)] def drift_correction(self, gap_width): if self.isOneGroup(): tmp_filter_list = self.get_filtered_points() startdate = tmp_filter_list.index[0] x_l = (tmp_filter_list.index[-1] - startdate).total_seconds() # y_n = y_0 + G(x_i / x_l) f = lambda row: row["DataValue"] + (gap_width * ( (row.name - startdate).total_seconds() / x_l)) tmp_filter_list["DataValue"] = tmp_filter_list.apply(f, axis=1) update_list = [{ "value": row["DataValue"], "id": index } for index, row in tmp_filter_list.iterrows()] ids = tmp_filter_list.index.tolist() self.memDB.update(update_list) self._populate_series() self.filtered_dataframe = self._series_points_df[ self._series_points_df.index.isin(ids)] return True return False def isOneGroup(self): issel = self._series_points_df.index.isin( self.get_filtered_points().index) found_group = False count = 0 for x in issel: if x: if not found_group: found_group = True count = count + 1 else: found_group = False if count > 1: return False if count == 1: return True def flag(self, qualifier_id): filtered_points = self.get_filtered_points() ''' query = "UPDATE DataValues SET QualifierID = %s WHERE ValueID = ?" % (qualifier_id) #self._cursor.executemany(query, [(str(x[0]),) for x in filtered_points]) self._cursor.executemany(query, [(str(x),) for x in filtered_points["ValueID"].astype(int).tolist()]) ''' self.memDB.updateFlag(filtered_points.index.tolist(), qualifier_id) ################### # Save/Restore ################### def restore(self): self.memDB.rollback() self._populate_series() self.reset_filter() def updateSeries(self, var=None, method=None, qcl=None, is_new_series=False): """ :param var: :param method: :param qcl: :param is_new_series: :return: """ var_id = var.id if var is not None else None method_id = method.id if method is not None else None qcl_id = qcl.id if qcl is not None else None #self.memDB.changeSeriesIDs(var_id, method_id, qcl_id) dvs = self.memDB.getDataValuesDF() if var_id is not None: dvs["VariableID"] = var_id if method_id is not None: dvs["MethodID"] = method_id if qcl_id is not None: dvs["QualityControlLevelID"] = qcl_id #if is new series remove valueids #if is_new_series: dvs["ValueID"] = None ''' for dv in dvs: dv.id = None ''' series = self.memDB.series_service.get_series_by_id(self._series_id) logger.debug("original editing series id: %s" % str(series.id)) if (var or method or qcl): tseries = self.memDB.series_service.get_series_by_id_quint( site_id=int(series.site_id), var_id=var_id if var else int(series.variable_id), method_id=method_id if method else int(series.method_id), source_id=series.source_id, qcl_id=qcl_id if qcl else int(series.quality_control_level_id)) if tseries: logger.debug("Save existing series ID: %s" % str(series.id)) series = tseries else: print "Series doesn't exist (if you are not, you should be running SaveAs)" if is_new_series: series = series_module.copy_series(series) if var: series.variable_id = var_id series.variable_code = var.code series.variable_name = var.name series.speciation = var.speciation series.variable_units_id = var.variable_unit_id series.variable_units_name = var.variable_unit.name series.sample_medium = var.sample_medium series.value_type = var.value_type series.time_support = var.time_support series.time_units_id = var.time_unit_id series.time_units_name = var.time_unit.name series.data_type = var.data_type series.general_category = var.general_category if method: series.method_id = method_id series.method_description = method.description if qcl: series.quality_control_level_id = qcl_id series.quality_control_level_code = qcl.code ''' dvs["LocalDateTime"] = pd.to_datetime(dvs["LocalDateTime"]) dvs["DateTimeUTC"] = pd.to_datetime(dvs["DateTimeUTC"]) ''' form = "%Y-%m-%d %H:%M:%S" series.begin_date_time = datetime.datetime.strptime( str(np.min(dvs["LocalDateTime"])), form) #np.min(dvs["LocalDateTime"])#dvs[0].local_date_time series.end_date_time = datetime.datetime.strptime( str(np.max(dvs["LocalDateTime"])), form) #np.max(dvs["LocalDateTime"])#dvs[-1].local_date_time series.begin_date_time_utc = datetime.datetime.strptime( str(np.min(dvs["DateTimeUTC"])), form) #dvs[0].date_time_utc series.end_date_time_utc = datetime.datetime.strptime( str(np.max(dvs["DateTimeUTC"])), form) #dvs[-1].date_time_utc series.value_count = len(dvs) ## Override previous save if not is_new_series: # delete old dvs #pass self.memDB.series_service.delete_values_by_series(series) #logger.debug("series.data_values: %s" % ([x for x in series.data_values])) dvs.drop('ValueID', axis=1, inplace=True) return series, dvs def save(self): """ Save to an existing catalog :param var: :param method: :param qcl: :return: """ series, dvs = self.updateSeries(is_new_series=False) if self.memDB.series_service.save_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Save was unsuccessful") return False def save_as(self, var=None, method=None, qcl=None): """ :param var: :param method: :param qcl: :return: """ series, dvs = self.updateSeries(var, method, qcl, is_new_series=True) if self.memDB.series_service.save_new_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Save As Function was Unsuccessful") return False def save_existing(self, var=None, method=None, qcl=None): """ :param var: :param method: :param qcl: :return: """ series, dvs = self.updateSeries(var, method, qcl, is_new_series=False) if self.memDB.series_service.save_series(series, dvs): logger.debug("series saved!") return True else: logger.debug("The Save As Existing Function was Unsuccessful") return False def create_qcl(self, code, definition, explanation): return self.memDB.series_service.create_qcl(code, definition, explanation) def create_method(self, description, link): return self.memDB.series_service.create_method(description, link) def create_qualifier(self, code, definition): return self.memDB.series_service.create_qualifier(code, definition) def create_variable(self, code, name, speciation, variable_unit_id, sample_medium, value_type, is_regular, time_support, time_unit_id, data_type, general_category, no_data_value): return self.memDB.series_service.create_variable( code, name, speciation, variable_unit_id, sample_medium, value_type, is_regular, time_support, time_unit_id, data_type, general_category, no_data_value) def reconcile_dates(self, parent_series_id): # FUTURE FEATURE: pull in new field data from another series and add to this series # (i.e one series contains new field data of an edited series at a higher qcl) pass
class TestPnlDataTable: def setup(self): #set up remote Database self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.dvs_size = 100 self.series = test_util.add_series_bulk_data(self.session, dvs_size=self.dvs_size) assert self.series assert len(self.series.data_values) == self.dvs_size self.memory_database = MemoryDatabase() self.memory_database.set_series_service(self.series_service) self.memory_database.initEditValues(self.series.id) self.app = wx.App() self.frame = wx.Frame(None) self.dataTable = FrmDataTable(self.frame) def test_build_series(self): dvs = self.session.query(DataValue).all() assert len(dvs) == self.dvs_size dvs = self.memory_database.mem_service._edit_session.query(DataValue).all() assert len(dvs) == self.dvs_size def test_get_data_values_data_frame(self): df = self.memory_database.getDataValuesDF() assert not df.empty def test_init_pnlDataTable(self): assert self.frame assert self.dataTable self.dataTable.init(self.memory_database) def test_selecting_points(self): self.dataTable.init(self.memory_database) values = self.dataTable.olvDataTable.dataframe assert not values.empty self.dataTable.onChangeSelection(values) myOlv = self.dataTable.olvDataTable count = 0 selected_item = myOlv.GetFirstSelected() assert selected_item != -1 # loop through selected items while selected_item != -1: selected_item = myOlv.GetNextSelected(selected_item) count += 1 assert count == self.dvs_size def test_deselecting_all(self): self.dataTable.init(self.memory_database) assert self.dataTable.olvDataTable.GetItemCount() == self.dvs_size values = self.dataTable.olvDataTable.dataframe self.dataTable.onChangeSelection(values) self.dataTable.olvDataTable.onDeselectAll() selected_item = self.dataTable.olvDataTable.GetFirstSelected() assert selected_item == -1 def test_clear_data_table(self): self.dataTable.init(self.memory_database) assert self.dataTable.olvDataTable.GetItemCount() == self.dvs_size self.dataTable.clear() assert not self.dataTable.olvDataTable.dataframe assert self.dataTable.olvDataTable.GetItemCount() == 0
class TestMemoryDB: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService( connection_string=self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) self.memory_db = MemoryDatabase() self.memory_db.set_series_service(self.series_service) self.series = test_util.add_series(self.session) self.memory_db.initEditValues(self.series.id) sorted_df = sorted(self.memory_db.df['LocalDateTime']) self.sdate = sorted_df[0] def test_get_data_values(self): dvs = self.memory_db.getDataValuesDF() assert len(dvs) == 10 def test_build_series(self): dvs = 100 self.series = test_util.add_series_bulk_data(self.session, dvs_size=dvs) assert self.series assert len(self.series.data_values) == dvs def test_update_points(self): self.memory_db.update([{"value": 15, "id": self.sdate}]) dvs = self.memory_db.getDataValuesDF() print dvs["DataValue"] assert dvs["DataValue"][0] == 15 def test_update_value(self): self.memory_db.updateValue([self.sdate], '+', 5) dvs = self.memory_db.getDataValuesDF() assert dvs["DataValue"][0] == 14 def test_add_points(self): #with pytest.raises(NotImplementedError): assert len(self.memory_db.getDataValuesDF().index) == 10 point = [('-9999', None, datetime.datetime(2011, 3, 25, 0, 0), '-7', datetime.datetime(2015, 3, 25, 7, 0), None, None, u'nc', None, None, self.series.site_id, self.series.variable_id, self.series.method_id, self.series.source_id, self.series.quality_control_level_id)] self.memory_db.addPoints(point) dvs = self.memory_db.getDataValuesDF() assert len(dvs.index) == 11 assert dvs["DataValue"][0] == -9999 def test_update_flag(self): self.memory_db.updateFlag([self.sdate], '50') dvs = self.memory_db.getDataValuesDF() assert dvs["QualifierID"][0] == '50' def test_delete_points(self): stlen = len(self.memory_db.df.index) self.memory_db.delete( self.memory_db.df["LocalDateTime"].tolist()[0:10]) dvs = self.memory_db.getDataValuesDF() assert len(dvs.index) == stlen - 10