def test_data_size_vs_diff(dm, given_dict, infer_dict): #Read all data from data model dm.read_data(normalize_data=False) #attr_list = [U_UNIVERSITY_CODE, PROGRAM_CODE, UNIVERSITY, MAJOR_CODE, TERM] attr_list = [U_UNIVERSITY_CODE, PROGRAM_CODE, UNIVERSITY] #attr_list = [MAJOR_CODE, PROGRAM_CODE, TERM] #Size of data data_size = len(dm.data) #Step size = 10 steps step_size = data_size//10 #Get experiment data in a dict size = [] accuracy = [] for i in xrange(step_size, data_size, step_size): dm_test = DataModel("") dm_test.set_data(dm.data[:i]) exp_test = Experimenter(dm_test, attr_list) actual = exp_test.get_actual_result(given_dict, infer_dict) estimation = exp_test.generic_get_estimated_result(given_dict, infer_dict) size.append(i) accuracy.append(abs(estimation - actual)) print("Step:%d--->Actual:%f--->Estimate:%f" %(i, actual, estimation)) print "-------------------------------------------------------------" plt.figure() plt.plot(size, accuracy) plt.title("Data Size vs Accuracy") plt.show()
def train_lstm(model_filename, weights_filename, l1_d=128, l2_d=128, b_s=128, bi=False): """ Trains a 2 layer lstm and saves the model in the files specified. Args: model_filename: The filename to save the model weights_filename: The filename to save the weights l1_d: Layer one dimensions l2_d: Layer two dimensions b_s: Batch size Returns: model: The lstm model fit on the training data """ dm=DataModel() texts, labels=dm.get_train_data() word_index, data=utils.get_word_index(texts) x_train, y_train, x_val, y_val, _, _=utils.get_train_val_test_data( data, labels) word_embeddings=utils.get_glove_embeddings() embedding_matrix=utils.get_embedding_matrix(word_embeddings, word_index) if bi: model=_bi_lstm(embedding_matrix, x_train, y_train, x_val, y_val, 3, word_index, l1_d, l2_d, b_s) else: model=_lstm(embedding_matrix, x_train, y_train, x_val, y_val, 3, word_index, l1_d, l2_d, b_s) save_model(model, model_filename, weights_filename) return model
class MainWindow(QMainWindow, Ui_MainWindow): def __init__(self, parent=None): super().__init__(parent) self.setupUi(self) self.mplWidget = MplWidget.warp_a_widget(self.matplotlibWidget) self.model = DataModel(self.mplWidget.canvas) self.tableView.setModel(self.model) validator = QDoubleValidator(0, 1000, 5) self.leastIntLineEdit.setValidator(validator) self.maxZDiffLineEdit.setValidator(validator) self.leastIntLineEdit.setText('200') self.maxZDiffLineEdit.setText('1') # Signals-Slots self.actionImportData.triggered.connect(self.setSampleFile) self.startFindButton.clicked.connect(self.startSearch) def setSampleFile(self): filename, _ = QFileDialog.getOpenFileName( self, "Input File", "", "Text files (*.txt);;All File(*)") if not filename: return self.model.setSample(filename) def startSearch(self): self.model.maxDiff = float(self.maxZDiffLineEdit.text()) self.model.minInt = float(self.leastIntLineEdit.text()) self.model.search()
def run_lstm_on_test_data(model_filename, weights_filename, kaggle_filename): dm = DataModel() test_x, test_id = dm.get_test_data() print(test_x.shape) _, data = utils.get_word_index(test_x) # word_embeddings = utils.get_glove_embeddings() # _ = utils.get_embedding_matrix(word_embeddings, word_index) y = utils.load_model_and_evaluate(model_filename, weights_filename, data) utils.write_kaggle_file(y, test_id, kaggle_filename)
def main(args): dm = DataModel(args.gig_file, args.chat_file) dm.read_data() exp = Experimenter(dm) if args.classify is True: scores = exp.classify_gigs() if args.feature_values is True: scores = exp.evaluate_feature_values() return dm
def main(argv): train_count = -1 if(len(argv)>0): train_count = int(argv[0]) dm = DataModel() dm.get_data(train_count) # (training, feature_names) = get_rich_featured_training(dm,lines) print(len(dm.data.keys())) print(len(dm.train)) print(len(dm.test))
def main(argv): train_count = -1 if (len(argv) > 0): train_count = int(argv[0]) dm = DataModel() dm.get_data(train_count) # (training, feature_names) = get_rich_featured_training(dm,lines) print(len(dm.data.keys())) print(len(dm.train)) print(len(dm.test))
def main(args): dm = DataModel(args.train_file) dm.read_train_data() exp = Experimenter(dm) distances = [x.get_distance() for x in dm.data] print(max(distances)) print(min(distances)) print(stats.mean(distances)) t1 = time.time() t2 = time.time() timeused = t2 - t1 logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \ (timeused/3600, timeused/60, timeused%60)) return exp
def accuracy_test(model_filename, weights_filename): dm = DataModel() x_train, y_train = dm.get_train_data() train_samples = int(0.8 * len(x_train)) x_test = x_train[train_samples:] y_test = y_train[train_samples:] _, data = utils.get_word_index(x_test) labels = to_categorical(np.asarray(y_test)) # word_embeddings = utils.get_glove_embeddings() # _ = utils.get_embedding_matrix(word_embeddings, word_index) md = utils.load_model(model_filename, weights_filename) result = md.evaluate(data, labels) print('\nTest loss:', result[0]) print('Test accuracy:', result[1])
def listevents(self, outputfile): """ Get common DB entries for table ipmievents, stored by captureevents. :param: outputfile - JSON output file, if None - special format to stdout. :returns: status, reply :raises: None """ table_name = 'ipmievents' if outputfile is not None: reply = "{'" + table_name + "':[" else: reply = '' sqlcommand = "SELECT id, datetime, event, details, status from {0}".format(table_name) try: ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand) except sqlalchemy.exc.OperationalError as ex: self.logger.error(traceback.format_exc()) if outputfile is not None: reply = reply + '{"exception":' + str(ex) + '}' except Exception as ex: self.logger.error(traceback.format_exc()) if outputfile is not None: reply = reply + '{"exception":' + str(ex) + '}' # Print out the ipmievents data in the common database. first = True ipmidata_keys = ipmidata.keys() for entry in ipmidata: if outputfile is not None: reply = reply + ("\n {" if first else "},\n {") first = second = False for v in ipmidata_keys: reply = ((reply + ",") if second else reply) + "'" + str(v) + "':'" + str(entry[v]) + "'" second = True self.logger.info(entry) else: print(entry[0], '|', entry[1], '|', entry[2], '|', entry[3], '|', entry[4]) if outputfile is not None: if ipmidata is not None: reply = reply + '}' if outputfile is not None: reply = reply + "\n]}" self.logger.info("status=0 json={0}".format(reply)) return 0, reply
def __init__(self, parent=None): super().__init__(parent) self.setupUi(self) self.mplWidget = MplWidget.warp_a_widget(self.matplotlibWidget) self.model = DataModel(self.mplWidget.canvas) self.tableView.setModel(self.model) validator = QDoubleValidator(0, 1000, 5) self.leastIntLineEdit.setValidator(validator) self.maxZDiffLineEdit.setValidator(validator) self.leastIntLineEdit.setText('200') self.maxZDiffLineEdit.setText('1') # Signals-Slots self.actionImportData.triggered.connect(self.setSampleFile) self.startFindButton.clicked.connect(self.startSearch)
def deleteevents(self, ids=None, outputfile=None): status = 0 table_name = 'ipmievents' # delet all event records if no ids provided sqlcommand = "DELETE from {0}".format(table_name) # delete selected event records if ids is not None: sqlcommand += " WHERE id IN ({0})".format(ids) if outputfile is not None: reply = "{'" + table_name + "':[" else: reply = '' try: ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand) except sqlalchemy.exc.OperationalError as ex: self.logger.error(traceback.format_exc()) if outputfile is not None: reply = reply + '{"exception":' + str(ex) + '}' status = 1 except Exception as ex: self.logger.error(traceback.format_exc()) if outputfile is not None: reply = reply + '{"exception":' + str(ex) + '}' status = 1 if outputfile is not None: reply = reply + "\n]}" self.logger.info("status=0 json={0}".format(reply)) return status, reply
def perform_datasize_vs_efficiency(self, given_dict, infer_dict, max_datasize=None, steps=10): sizes, est_times, acc_times = [], [], [] if max_datasize is None: max_datasize = len(self.dm.data) data_step = max_datasize / steps for i in range(steps): cur_datasize = (i+1) * data_step data = self.dm.data while len(data) < cur_datasize: data.extend(self.dm.data) cur_data = data[:cur_datasize] cur_dm = DataModel("") cur_dm.set_data(cur_data) cur_exp = Experimenter(cur_dm, self.attr_list) (cur_est, cur_acc) = cur_exp.time_n_queries(given_dict, infer_dict) sizes.append(cur_datasize) est_times.append(float(sum(cur_est))/len(cur_est)) acc_times.append(float(sum(cur_acc))/len(cur_acc)) return (sizes, est_times, acc_times)
def generate_w2c_word_embeddings(): """ Generates word 2 vector embeddings. """ dm = DataModel() train_x, _ = dm.get_train_data() w2c = Word2Vec(method="skipgram", corpus=list(train_x), window_size=5, n_hidden=128, n_epochs=3, learning_rate=0.08) W1, W2, loss_vs_epoch = w2c.run() pkl_dump = [W1, W2, loss_vs_epoch] with open('embeddings.pickle', 'wb') as handle: pickle.dump(pkl_dump, handle)
def OnOpenImage(self, evt=None): # 1) Get the image key # Start with the table_id if there is one tblNum = None if p.table_id: dlg = wx.TextEntryDialog(self, p.table_id + ':', 'Enter ' + p.table_id) dlg.SetValue('0') if dlg.ShowModal() == wx.ID_OK: try: tblNum = int(dlg.GetValue()) except ValueError: errdlg = wx.MessageDialog( self, 'Invalid value for %s!' % (p.table_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION) errdlg.ShowModal() return dlg.Destroy() else: dlg.Destroy() return # Then get the image_id dlg = wx.TextEntryDialog(self, p.image_id + ':', 'Enter ' + p.image_id) dlg.SetValue('') if dlg.ShowModal() == wx.ID_OK: try: imgNum = int(dlg.GetValue()) except ValueError: errdlg = wx.MessageDialog( self, 'Invalid value for %s!' % (p.image_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION) errdlg.ShowModal() return dlg.Destroy() else: dlg.Destroy() return # Build the imkey if p.table_id: imkey = (tblNum, imgNum) else: imkey = (imgNum, ) dm = DataModel.getInstance() if imkey not in dm.GetAllImageKeys(): errdlg = wx.MessageDialog(self, 'There is no image with that key.', "Couldn't find image", wx.OK | wx.ICON_EXCLAMATION) errdlg.ShowModal() self.Destroy() else: # load the image self.img_key = imkey self.SetImage(imagetools.FetchImage(imkey), p.image_channel_colors) self.DoLayout()
def export(items, tempdir): """Export a list of items Arguments items -- list of items to export tempdir -- directory to use for the export operation """ initdir = spm.spmanager.getFirstPath([spm.ExportFolder, spm.ImportFolder, spm.MostRecentFolder]) filenamepath = tkFileDialog.asksaveasfilename(initialdir = initdir, filetypes = ff.dlgExportFormats, defaultextension = ff.dlgDefaultExportExt) if(len(filenamepath) < 1): return spm.spmanager.setPath(spm.ExportFolder, os.path.dirname(filenamepath)) #Create export dir and datamodel dmdir = os.path.join(tempdir, _exportdir) if(os.path.exists(dmdir)): shutil.rmtree(dmdir) os.makedirs(dmdir) dm = DataModel(dmdir) #Add all slideshows for item in items: if(not dm.addSlideshow(item, True)): showerror(lang[lng.txtExportError], lang[lng.txtCouldNotExport] + item.title) shutil.rmtree(dmdir) return #Save and zip dm.saveToFile() pack(dmdir, filenamepath) shutil.rmtree(dmdir)
def main(args): dm = DataModel(args.data_file) dm.read_data(to_read_count=10000) exp = Experimenter(dm, \ process_datamodel=True, \ serialise=False) t1 = time.time() exp.perform_multiclass_experiment( pred_mode=INDEPENDENT, use_exclusion=True, need_to_extract_features=True, prediction_file='../results/predictions_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.csv', result_file='../results/results_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.txt', english_only=True, legible_only=True) t2 = time.time() timeused = t2 - t1 logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \ (timeused/3600, timeused/60, timeused%60)) return exp
def OnOpenImage(self, evt=None): # 1) Get the image key # Start with the table_id if there is one tblNum = None if p.table_id: dlg = wx.TextEntryDialog(self, p.table_id + ":", "Enter " + p.table_id) dlg.SetValue("0") if dlg.ShowModal() == wx.ID_OK: try: tblNum = int(dlg.GetValue()) except ValueError: errdlg = wx.MessageDialog( self, "Invalid value for %s!" % (p.table_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION ) errdlg.ShowModal() return dlg.Destroy() else: dlg.Destroy() return # Then get the image_id dlg = wx.TextEntryDialog(self, p.image_id + ":", "Enter " + p.image_id) dlg.SetValue("") if dlg.ShowModal() == wx.ID_OK: try: imgNum = int(dlg.GetValue()) except ValueError: errdlg = wx.MessageDialog( self, "Invalid value for %s!" % (p.image_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION ) errdlg.ShowModal() return dlg.Destroy() else: dlg.Destroy() return # Build the imkey if p.table_id: imkey = (tblNum, imgNum) else: imkey = (imgNum,) dm = DataModel.getInstance() if imkey not in dm.GetAllImageKeys(): errdlg = wx.MessageDialog( self, "There is no image with that key.", "Couldn't find image", wx.OK | wx.ICON_EXCLAMATION ) errdlg.ShowModal() self.Destroy() else: # load the image self.img_key = imkey self.SetImage(imagetools.FetchImage(imkey), p.image_channel_colors) self.DoLayout()
def updateTargetHBAs(self): self.logger.info("ENTERED updateTargetHBAs") try: db = DataModel() table = 'target_hbas' self.logger.info("clearing table: {0}".format(table)) cmd = 'DELETE FROM {0} WHERE nodeId={1}'.format(table, self.nodeID) db.ExecuteRawQueryStatement(cmd) for wwn in self.__get_target_hba_wwns(): cmd = ("INSERT INTO {0} (wwn, created_at, updated_at, nodeId) VALUES ('{1}'," "(select datetime('now')), (select datetime('now')), {2});".format(table, wwn, self.nodeID)) self.logger.info("Executing the following command: {0}".format(cmd)) db.ExecuteRawQueryStatement(cmd) self.logger.info("Exiting updateTargetHBAs") return True except Exception as ex: self.logger.error("An exception occurred while adding new wwn to TargetHBA table: {0}".format(ex)) return False
def main(args): dm = DataModel(args.data_file) dm.read_data(to_read_count=10000) exp = Experimenter(dm, \ process_datamodel=True, \ serialise=False) t1 = time.time() exp.perform_multiclass_experiment( pred_mode=INDEPENDENT, use_exclusion=True, need_to_extract_features=True, prediction_file= '../results/predictions_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.csv', result_file= '../results/results_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.txt', english_only=True, legible_only=True) t2 = time.time() timeused = t2 - t1 logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \ (timeused/3600, timeused/60, timeused%60)) return exp
def perform_datasize_vs_accuracy(self, given_dict, infer_dict, max_datasize=None, steps=10): #Get experiment data in a dict size = [] accuracy = [] if max_datasize is None: max_datasize = len(self.dm.data) data_step = max_datasize / steps for i in range(steps): cur_datasize = (i+1) * data_step data = self.dm.data while len(data) < cur_datasize: data.extend(self.dm.data) cur_data = data[:cur_datasize] cur_dm = DataModel("") cur_dm.set_data(cur_data) cur_exp = Experimenter(cur_dm, self.attr_list) actual = cur_exp.get_actual_result(given_dict, infer_dict) estimation = cur_exp.generic_get_estimated_result(given_dict, infer_dict) size.append(cur_datasize) accuracy.append(abs(estimation - actual)) return (size, accuracy)
def listhbas(): """ \b NAME listhbas - will show a list of hbas SYNOPSIS listhbas [OPTIONS] DESCRIPTION Will list out available hbas seen by the LightSpeed No options are available for this command """ targetHBAStable = DataModel() cmd = "SELECT id,wwn from target_hbas" q = targetHBAStable.GetListOfQuery(cmd) cli_logger.info('List of available target HBAs: ') cli_logger.info( '---------------------------------------------------------------------------' ) for row in q: cli_logger.info('ID: {0} | HBA: {1}'.format(row[0], row[1])) cli_logger.info( '---------------------------------------------------------------------------' )
def get_image_keys_at_row(self, row): '''Returns a list of image keys at the given row or None if the column names can't be found in col_labels ''' if self.key_indices is None or self.grouping is None: return None else: if self.grouping.lower() == 'image': return [tuple(self.data[self.row_order,:][row, self.key_indices])] elif self.grouping.lower() == 'object': return [tuple(self.data[self.row_order,:][row, self.key_indices[:-1]])] else: dm = DataModel.getInstance() return dm.GetImagesInGroup(self.grouping, self.get_row_key(row))
def PerImageCounts(self, filter_name=None, cb=None): # Clear the current perClassObjects storage for bin in self.classBins: self.perClassObjects[bin.label] = [] # Retrieve a data model instance dm = DataModel.getInstance() # Retrieve image keys and initialize variables imageKeys = dm.GetAllImageKeys(filter_name) imageAmount = float(len(imageKeys)) perImageData = [] # Process all images for k_index, imKey in enumerate(imageKeys): try: # Retrieve the keys of the objects in the current image obKeys = dm.GetObjectsFromImage(imKey) except: raise 'No such image: %s' % (imKey, ) return # Calculate the amount of hits for each of the classes in the current image classHits = {} objectCount = [imKey[0]] if obKeys: classObjects = self.FilterObjectsFromClassN(keys=[imKey]) for clNum, bin in enumerate(self.classBins): # Get the objects from the image which belong to the selected class classHits[bin.label] = classObjects[float(clNum + 1)] # Store the total object count of this class for the current image nrHits = len(classHits[bin.label]) objectCount.append(nrHits) # Store the objects for the current class and image grouped # by class if any are found for this class in the selected image if nrHits > 0: self.perClassObjects[bin.label] += classHits[bin.label] else: # If there are objects in the image, add zeros for all bins [objectCount.append(0) for bin in self.classBins] # Store the results for the current image and update the callback # function if available perImageData.append(objectCount) if cb: cb(min(1, k_index / imageAmount)) return perImageData
def PerImageCounts(self, filter_name=None, cb=None): # Clear the current perClassObjects storage for bin in self.classBins: self.perClassObjects[bin.label] = [] # Retrieve a data model instance dm = DataModel.getInstance() # Retrieve image keys and initialize variables imageKeys = dm.GetAllImageKeys(filter_name) imageAmount = float(len(imageKeys)) perImageData = [] # Process all images for k_index, imKey in enumerate(imageKeys): try: # Retrieve the keys of the objects in the current image obKeys = dm.GetObjectsFromImage(imKey) except: raise ValueError('No such image: %s' % (imKey,)) # Calculate the amount of hits for each of the classes in the current image classHits = {} objectCount = [imKey[0]] if obKeys: classObjects = self.FilterObjectsFromClassN(keys = [imKey]) for clNum, bin in enumerate(self.classBins): # Get the objects from the image which belong to the selected class classHits[bin.label] = classObjects[float(clNum+1)] # Store the total object count of this class for the current image nrHits = len(classHits[bin.label]) objectCount.append(nrHits) # Store the objects for the current class and image grouped # by class if any are found for this class in the selected image if nrHits > 0: self.perClassObjects[bin.label] += classHits[bin.label] else: # If there are objects in the image, add zeros for all bins [objectCount.append(0) for bin in self.classBins] # Store the results for the current image and update the callback # function if available perImageData.append(objectCount) if cb: cb(min(1, k_index/imageAmount)) return perImageData
def updateInitiatorHBAs(self): # This function will update all HBA related information. There are two types present: # a) Target HBA's # QLE which will operate in Target mode. Target HBA's will need to be configured in database # and LIO config. # b) Initiator HBA's database entries only. (Some could be QLE.) self.logger.info("ENTERED updateInitiatorHBAs") try: table = 'initiator_hbas' db = DataModel() self.logger.info("clearing table: {0}".format(table)) cmd = 'DELETE FROM {0} WHERE nodeId={1}'.format(table, self.nodeID) db.ExecuteRawQueryStatement(cmd) self.logger.info("Now adding the provided wwns.") for wwn in self.__get_initiator_hba_wwns(): self.logger.info("Adding the wwn {0} to the table".format(wwn)) cmd = ("INSERT INTO {0} (wwn, created_at, updated_at, nodeId) VALUES ('{1}', (select datetime('now')), " "(select datetime('now')), {2});".format(table, wwn, self.nodeID)) self.logger.info("Executing the command: {0}".format(cmd)) db.ExecuteRawQueryStatement(cmd) return True except Exception as ex: self.logger.error("An exception occurred while updating the initiator table: {0}".format(ex)) return False
def sel_timeupdated(self): # Update the timeupdated table with domain ipmievents for datetime now. domain_name = 'ipmievents' table_name = 'timeupdated' sqlcommand = ("UPDATE {0} SET datetime=DATETIME('now') WHERE domain='{1}'" .format(table_name, domain_name)) try: ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand) except sqlalchemy.exc.OperationalError as ex: self.logger.error(traceback.format_exc()) print('{"exception":' + str(ex) + '}') return 1 except Exception as ex: self.logger.error(traceback.format_exc()) print('{"exception":' + str(ex) + '}') return 1 return 0
def get_object_keys_at_row(self, row): '''Returns a list of object keys at the given row or None if the column names can't be found in col_labels ''' if self.key_indices is None or self.grouping is None: return None else: dm = DataModel.getInstance() # If the key index for the row is an object key, just return that key if self.grouping.lower() == 'object': return [tuple(self.data[self.row_order,:][row, self.key_indices])] else: # Otherwise, return all object keys in the image imkeys = self.get_image_keys_at_row(row) obkeys = [] for imkey in imkeys: obs = dm.GetObjectCountFromImage(imkey) obkeys += [tuple(list(imkey)+[i]) for i in range(1,obs+1)] return obkeys
def on_dclick_label(self, evt): '''Handle display of images and objects''' if evt.Row >= 0: obkeys = self.grid.Table.get_object_keys_at_row(evt.Row) if self.grid.Table.grouping is None: # We need to know how the table is grouped to know what to do logging.warn( 'CPA does not know how to link this table to your images. Can\'t launch ImageViewer.' ) return elif self.grid.Table.grouping.lower() == 'object': # For per-object grouping, show the objects in the image imview = imagetools.ShowImage(obkeys[0][:-1], p.image_channel_colors, parent=self.Parent) if obkeys is not None: for obkey in obkeys: imview.SelectObject(obkey) elif self.grid.Table.grouping.lower() == 'image': # For per-image grouping just show the images. # If there is only one object, then highlight it if obkeys is not None and len(obkeys) == 1: imview = imagetools.ShowImage(obkeys[0][:-1], p.image_channel_colors, parent=self.Parent) imview.SelectObject(obkeys[0]) else: imkeys = self.grid.Table.get_image_keys_at_row(evt.Row) if imkeys: #XXX: warn if there are a lot for imkey in imkeys: imagetools.ShowImage(imkey, p.image_channel_colors, parent=self.Parent) else: key_cols = self.grid.Table.get_row_key(evt.Row) if key_cols: dm = DataModel.getInstance() imkeys = dm.GetImagesInGroup(self.grid.Table.grouping, key_cols) for imkey in imkeys: imagetools.ShowImage(imkey, p.image_channel_colors, parent=self.Parent)
def __init__(self): self._dm = DataModel() # Primary data model. self._topics = [] # All topics for which judgments have been loaded. self._documents = [] # Documents for which we have a judgment for the # currently selected topic. self._selected_topic = None # Currently selected topic. self._selected_document = None # Currently selected document. self._rationales = [] # Rationales for the currently selected document. self._display_text = None # Text of document being manipulated. super(CWR, self).__init__() self.init_UI() # For testing WebView. '''
def get_image_keys_at_row(self, row): '''Returns a list of image keys at the given row or None if the column names can't be found in col_labels ''' if self.key_indices is None or self.grouping is None: return None else: if self.grouping.lower() == 'image': return [ tuple(self.data[self.row_order, :][row, self.key_indices]) ] elif self.grouping.lower() == 'object': return [ tuple(self.data[self.row_order, :][row, self.key_indices[:-1]]) ] else: dm = DataModel.getInstance() return dm.GetImagesInGroup(self.grouping, self.get_row_key(row))
def on_dclick_label(self, evt): '''Handle display of images and objects''' if evt.Row >= 0: obkeys = self.grid.Table.get_object_keys_at_row(evt.Row) if self.grid.Table.grouping is None: # We need to know how the table is grouped to know what to do logging.warn('CPA does not know how to link this table to your images. Can\'t launch ImageViewer.') return elif self.grid.Table.grouping.lower() == 'object': # For per-object grouping, show the objects in the image imview = imagetools.ShowImage(obkeys[0][:-1], p.image_channel_colors, parent=self.Parent) if obkeys is not None: for obkey in obkeys: imview.SelectObject(obkey) elif self.grid.Table.grouping.lower() == 'image': # For per-image grouping just show the images. # If there is only one object, then highlight it if obkeys is not None and len(obkeys) == 1: imview = imagetools.ShowImage(obkeys[0][:-1], p.image_channel_colors, parent=self.Parent) imview.SelectObject(obkeys[0]) else: imkeys = self.grid.Table.get_image_keys_at_row(evt.Row) if imkeys: #XXX: warn if there are a lot for imkey in imkeys: imagetools.ShowImage(imkey, p.image_channel_colors, parent=self.Parent) else: key_cols = self.grid.Table.get_row_key(evt.Row) if key_cols: dm = DataModel.getInstance() imkeys = dm.GetImagesInGroup(self.grid.Table.grouping, key_cols) for imkey in imkeys: imagetools.ShowImage(imkey, p.image_channel_colors, parent=self.Parent)
def get_object_keys_at_row(self, row): '''Returns a list of object keys at the given row or None if the column names can't be found in col_labels ''' if self.key_indices is None or self.grouping is None: return None else: dm = DataModel.getInstance() # If the key index for the row is an object key, just return that key if self.grouping.lower() == 'object': return [ tuple(self.data[self.row_order, :][row, self.key_indices]) ] else: # Otherwise, return all object keys in the image imkeys = self.get_image_keys_at_row(row) obkeys = [] for imkey in imkeys: obs = dm.GetObjectCountFromImage(imkey) obkeys += [ tuple(list(imkey) + [i]) for i in range(1, obs + 1) ] return obkeys
def FilterObjectsFromClassN(self, classN = None, keys = None): ''' Filter the input objects to output the keys of those in classN, using a defined SVM model classifier. ''' # Retrieve instance of the database connection db = dbconnect.DBConnect.getInstance() object_data = {} if isinstance(keys, str): object_data[0] = db.GetCellDataForClassifier(keys) elif keys != []: if len(keys) == len(dbconnect.image_key_columns()): # Retrieve instance of the data model and retrieve objects in the requested image dm = DataModel.getInstance() obKeys = dm.GetObjectsFromImage(keys[0]) else: obKeys = keys for key in obKeys: object_data[key] = db.GetCellDataForClassifier(key) sorted_keys = sorted(object_data.keys()) values_array = np.array([object_data[key] for key in sorted_keys]) scaled_values = self.ScaleData(values_array) pred_labels = self.model.predict(scaled_values) # Group the object keys per class classObjects = {} for index in range(1, len(self.classBins)+1): classObjects[float(index)] = [] for index, label in enumerate(pred_labels): classObjects[np.int(label)+1].append(sorted_keys[index]) # Return either a summary of all classes and their corresponding objects # or just the objects for a specific class if classN is None: return classObjects else: return classObjects[classN]
def FilterObjectsFromClassN(self, classN=None, keys=None): ''' Filter the input objects to output the keys of those in classN, using a defined SVM model classifier. ''' # Retrieve instance of the database connection db = dbconnect.DBConnect.getInstance() object_data = {} if isinstance(keys, str): object_data[0] = db.GetCellDataForClassifier(keys) elif keys != []: if len(keys) == len(dbconnect.image_key_columns()): # Retrieve instance of the data model and retrieve objects in the requested image dm = DataModel.getInstance() obKeys = dm.GetObjectsFromImage(keys[0]) else: obKeys = keys for key in obKeys: object_data[key] = db.GetCellDataForClassifier(key) sorted_keys = sorted(object_data.keys()) values_array = np.array([object_data[key] for key in sorted_keys]) scaled_values = self.ScaleData(values_array) pred_labels = self.model.predict(scaled_values) # Group the object keys per class classObjects = {} for index in range(1, len(self.classBins) + 1): classObjects[float(index)] = [] for index, label in enumerate(pred_labels): classObjects[np.int(label) + 1].append(sorted_keys[index]) # Return either a summary of all classes and their corresponding objects # or just the objects for a specific class if classN is None: return classObjects else: return classObjects[classN]
def getconfiguration(self): """ Fetch IPMI configuration from database :param: none :returns: status, reply :raises: None """ status = 1 obj = PxJSON("Unable to obtain IPMI configuration") try: table_name = 'systemsetups' res = DataModel().ExecuteRawQueryStatement("SELECT ipmi_connection_type, ipmi_address, ipmi_netmask, ipmi_gateway, ipmi_vlan from {0}".format(table_name)) reply = {} for row in res: self.logger.info(row) reply[PxJSON.CONNECTIONTYPE] = row['ipmi_connection_type'] reply[PxJSON.IPV4] = row['ipmi_address'] reply[PxJSON.NETMASK] = row['ipmi_netmask'] reply[PxJSON.GATEWAY] = row['ipmi_gateway'] reply[PxJSON.VLAN] = row['ipmi_vlan'] if row['ipmi_vlan'] != 0 else 'undefined' obj.setroute(PxJSON.IPMI_INFO, reply) obj.setsuccess() status = 0 except sqlalchemy.exc.OperationalError as ex: self.logger.error(traceback.format_exc()) obj.internal({"exception": str(ex)}) except Exception as ex: self.logger.error(traceback.format_exc()) obj.internal({"exception": str(ex)}) self.logger.info("status={0} json={1}".format(status, obj.getjsonpretty())) return status, obj.getjson()
def get_object_keys_at_row(self, row): # XXX: needs to be updated to work for per_well data if self.table_name == p.image_table: # return all objects in this image key = self.get_row_key(row) if key is None: return None dm = DataModel.getInstance() n_objects = dm.GetObjectCountFromImage(key) return [tuple(list(key) + [i]) for i in range(n_objects)] elif self.table_name == p.object_table: key = self.get_row_key(row) if key is None: return None return [key] else: key = [] for col in dbconnect.object_key_columns(): if col not in self.col_labels: return None else: col_index = self.col_labels.tolist().index(col) key += [self.GetValue(row, col_index)] return [tuple(key)]
self.tile_collection.cv.acquire() heappush(self.tile_collection.loadq, ((0, 0, 0), '<ABORT>')) self.tile_collection.cv.notify() self.tile_collection.cv.release() ################# FOR TESTING ########################## if __name__ == "__main__": app = wx.PySimpleApp() from datamodel import DataModel p = Properties.getInstance() p.LoadFile('../properties/nirht_test.properties') db = DBConnect.getInstance() db.connect() dm = DataModel.getInstance() test = TileCollection.getInstance() f = wx.Frame(None) for i in xrange(10): obKey = dm.GetRandomObject() test.GetTileData((0, 1, 1), f) for t in threading.enumerate(): if t != threading.currentThread(): t.abort() f.Destroy() app.MainLoop()
class CWR(QtGui.QWidget): def __init__(self): self._dm = DataModel() # Primary data model. self._topics = [] # All topics for which judgments have been loaded. self._documents = [] # Documents for which we have a judgment for the # currently selected topic. self._selected_topic = None # Currently selected topic. self._selected_document = None # Currently selected document. self._rationales = [] # Rationales for the currently selected document. self._display_text = None # Text of document being manipulated. super(CWR, self).__init__() self.init_UI() # For testing WebView. ''' test_url = 'https://en.wikipedia.org/wiki/The_Beatles' content = requests.get(test_url).text content test_text = BeautifulSoup(content, "html.parser").get_text() self._display_text = test_text ''' def init_UI(self): grid = QtGui.QGridLayout() grid.setSpacing(10) ##################################### # Summary of UI Elements # ##################################### # These are UI elements that are updated after creation. self._confusion_matrix = None # String form of confusion matrix for current view. ##################################### # Topic View # ##################################### # Contains Topic List. topic_view = QtGui.QGroupBox("Topics") topic_layout = QtGui.QVBoxLayout() topic_view.setLayout(topic_layout) grid.addWidget(topic_view, 0, 0) # Topic List self._topic_list = QtGui.QListWidget() self._topic_list.setSizePolicy(QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Minimum) self._topic_list.itemClicked.connect(self._topic_selected) topic_layout.addWidget(self._topic_list) ##################################### # Document View # ##################################### # Contains Document List. document_view = QtGui.QGroupBox("Documents") document_layout = QtGui.QVBoxLayout() document_view.setLayout(document_layout) grid.addWidget(document_view, 0, 1) # Document List self._document_list = QtGui.QListWidget() self._document_list.setSizePolicy(QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding) self._document_list.itemClicked.connect(self._document_selected) document_layout.addWidget(self._document_list) ##################################### # Statistics View # ##################################### # Below document view. Contains the confusion matrix, list of # rationales, gold standard values, and user judgments. #stat_label = QtGui.QLabel() #stat_label.setText("<B>Statistics</B>") #stat_layout = QtGui.QVBoxLayout() #document_layout.addWidget(stat_label) #self._stat_display.setLayout(stat_layout) #self._stat_display.setSizePolicy(QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Minimum) # Topic Display topic = QtGui.QLabel("<b>Topic</b>: N/A") document_layout.addWidget(topic) narrative_label = QtGui.QLabel("<b>Narrative</b:") document_layout.addWidget(narrative_label) narrative_view = QtGui.QTextEdit() document_layout.addWidget(narrative_view) # Confusion Matrix for current Topic or Topic-Document Pair confusion_matrix_label = QtGui.QLabel() confusion_matrix_label.setText("Confusion Matrix") document_layout.addWidget(confusion_matrix_label) confusion_matrix = QtGui.QLabel() confusion_matrix.setText(" - - - - \n" " - - - - \n" " - - - - \n") document_layout.addWidget(confusion_matrix) # Gold Standard for current Topic-Document. gold_standard_view = QtGui.QLabel() gold_standard_view.setText("Gold Standard: N/A") document_layout.addWidget(gold_standard_view) # Degree 1 Agreement d1_agreement_view = QtGui.QLabel() d1_agreement_view.setText("D1 Agreement: N/A") document_layout.addWidget(d1_agreement_view) # Degree 2 Agreement d2_agreement_view = QtGui.QLabel() d1_agreement_view.setText("D2 Agreement: N/A") document_layout.addWidget(d2_agreement_view) # Give pointers to updateable elements. self._topic_view = topic self._narrative_view = narrative_view self._confusion_matrix = confusion_matrix self._gold_standard_view = gold_standard_view self._d1_agreement_view = d1_agreement_view self._d2_agreement_view = d2_agreement_view ##################################### # Rationale View # ##################################### # Contains the rationale check boxes, text display, and statistics. rationale_view = QtGui.QGridLayout() grid.addLayout(rationale_view, 0, 2) # Rationale Display display = HighlightWebView() display.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding) display.load(QUrl('https://en.wikipedia.org/wiki/The_Beatles')) display.show() self._rationale_display = display rationale_view.addWidget(self._rationale_display, 2, 0) # Rationale Selection selection_view = QtGui.QGroupBox() self._selection_layout = QtGui.QHBoxLayout() selection_view.setLayout(self._selection_layout) rationale_view.addWidget(selection_view, 1, 0) ##################################### # Worker View # ##################################### # Below rationale view. Contains pure text of worker rationales, # judgments, and the gold standard. self._worker_display = QtGui.QGroupBox("Workers") worker_layout = QtGui.QVBoxLayout() grid.addLayout(worker_layout, 0, 3) #self._worker_display.setLayout(worker_layout) # Worker Rationales for current Topic-Document. worker_rationale_list_label = QtGui.QLabel() worker_rationale_list_label.setText("Worker Rationales") worker_layout.addWidget(worker_rationale_list_label) worker_rationale_list = QtGui.QTextEdit() worker_rationale_list.setText("Initial Text.") worker_layout.addWidget(worker_rationale_list) # Worker Judgments for current Topic-Document. worker_judgment_list_label = QtGui.QLabel() worker_judgment_list_label.setText("Worker Judgments") worker_layout.addWidget(worker_judgment_list_label) worker_judgment_list = QtGui.QTextEdit() worker_judgment_list.setText("Initial Text.") worker_layout.addWidget(worker_judgment_list) # Give pointers to updateable elements. self._worker_judgments = worker_judgment_list self._worker_rationales = worker_rationale_list ##################################### # Main Application Properties # ##################################### self.setLayout(grid) self.setGeometry(300, 300, 1000, 1000) self.setWindowTitle("The Crowdworker's Rationale") self.show() def load(self, directory): ''' Loads rationale data from the specified file. ''' dm = self._dm for dirpath, _, filenames in os.walk(directory): for f in filenames: if f.endswith(".csv"): absolute_path = os.path.abspath(os.path.join(dirpath, f)) dm.load(absolute_path) self.update_topic_list(dm.judged_topics()) self.update_document_list([]) ######################################################################################### # Updateable UI Elements # ######################################################################################### def update_topic_list(self, topics): ''' Updates the topics displayed in Topic View. ''' self._topics = topics self._topic_list.clear() self._topic_list.addItems([t.id for t in topics]) self._topic_list.sortItems() def update_document_list(self, documents): ''' Updates the documents displayed in Document View. ''' self._document_list.clear() self._document_list.addItems(documents) self._document_list.sortItems() def update_rationale_selection(self, rationales): ''' Regenerates data structures and display logic for rationale selection. ''' # Compact holder for all rationale logic. container = namedtuple('RationaleContainer', ['rationale', 'color', 'display']) # Friendly display colors used to highlight rationales in source texts. colors = [QtGui.QColor(102, 255, 102), # Light Green QtGui.QColor(255, 102, 102), # Light Red QtGui.QColor(102, 201, 255), # Electric Blue QtGui.QColor(102, 178, 255), # Baby Blue QtGui.QColor(178, 102, 255), # Light Purple QtGui.QColor(255, 205, 255), # Light Magenta QtGui.QColor(255, 102, 178), # Light Pink QtGui.QColor(192, 192, 192)] # Light Grey # Remove old rationale widgets. layout = self._selection_layout for i in reversed(range(layout.count())): layout.itemAt(i).widget().setParent(None) # Re-generate rationale data structures. self._rationales = [] for i in range(len(rationales)): r = rationales[i] c = colors[min(i, len(colors) - 1)] d = QCheckBox(r.label) d.stateChanged.connect(self._rationale_selection_changed) self._rationales.append(container(rationale = r, color = c, display = d)) self._selection_layout.addWidget(d) def update_rationale_display(self): ''' Recomputes rationale overlap and updates display. Expensive. ''' print ("Updating rationale display...") # Disable rationale selection while updating. for this_rationale in self._rationales: this_rationale.display.setEnabled(False) # Locate selected rationales. selected = [] for this_rationale in self._rationales: if this_rationale.display.isChecked(): selected.extend([r for r in self._rationales if this_rationale.display.text() == r.rationale.label]) print (selected) # Gather text. text = self._rationale_display.get_text() # Compute overlap. just_rationales = [r.rationale for r in selected] result = Rationale.compute_overlap(text, just_rationales) # Update display with rationale matches. display = self._rationale_display display.clear() # If more than one rationale is selected, highlight overlap. ''' if result.overlap: for string in result.overlap: print ("Highlighted string: %s" % string) display.highlight(string) # Else, highlight single rationale elif result.matches: strings = [QtCore.QString(s) for s in itertools.chain(*[s for s in result.matches.values()])] for string in strings: print ("Highlighting string: %s" % string) display.highlight(string) ''' for r in just_rationales: display.highlight(r.rationale.rationale) # This is code exclusively for a multi-color highlight interface. ''' for rationale, matches in result.matches.items(): # Map back to container to find correct color. color = [c.color for c in selected if c.rationale is rationale][0] # Highlight in display. for match in matches: display.highlight(match, color) overlap_color = QtGui.QColor(255, 255, 102) # Light Yellow for string in result.overlap: display.highlight(string, overlap_color) ''' # Re-enable rationale selection. for this_rationale in self._rationales: this_rationale.display.setEnabled(True) def update_statistics(self, topic=None, document=None): self.update_gold_standard_view(topic, document) self.update_topic_view(topic) self.update_rationale_list() self.update_judgment_list() self.update_confusion_matrix(topic, document) self.update_agreement_view(topic, document) def update_confusion_matrix(self, topic=None, document=None): cm = self._dm.confusion_matrix(topic, document) self._confusion_matrix.setText(cm) def update_gold_standard_view(self, topic, document): ''' Updates the current gold standard view. ''' if topic and document: value = self._dm.gold_standard(topic, document) self._gold_standard_view.setText("Gold Standard: %s" % value) def update_topic_view(self, topic): ''' Updates the current topic and rationale display. ''' (topic, narrative) = self._dm.topic_information(topic) self._topic_view.setText("<b>Topic</b>: %s" % topic) self._narrative_view.setText("%s" % narrative) def update_agreement_view(self, topic, document): ''' Computes and updates the agreement for currently selected Topic or Document. ''' d1_agree = self._dm.agreement(1, topic, document) self._d1_agreement_view.setText("D1 Agreement: %f" % d1_agree) d2_agree = self._dm.agreement(2, topic, document) self._d2_agreement_view.setText("D2 Agreement: %f" % d2_agree) def update_rationale_list(self): ''' Updates the worker rationale list. If one or more worker IDs is selected, this will display only the rationales by the selected workers. Otherwise, this will display all rationales for the selected Topic and Document. ''' display_text = '' selected = [r for r in self._rationales if r.display.isChecked()] # If none are selected, display all. selected = selected if selected else self._rationales for r in selected: display_text += ('%s\n\n%s\n\n' % (r.rationale.label, r.rationale.rationale.rationale)) self._worker_rationales.setText(display_text) def update_judgment_list(self, topic=None, document=None): ''' Updates the worker judgment list. If one or more worker IDs is selected, this will display only the judgments of those selected workers. Otherwise, this will display the judgments from all workers. ''' display_text = '' selected = [r for r in self._rationales if r.display.isChecked()] # If none are selected, display all. selected = selected if selected else self._rationales for r in selected: display_text += ('%s: %s\n' % (r.rationale.label, r.rationale.rationale.value)) self._worker_judgments.setText(display_text) def update_rationale_text(self, text): ''' This updates the text in the rationale display. ''' self._rationale_display.set_text(text) def highlight_rationale(self, text): self._rationale_display.highlight(text) def load_document(self, url): self._rationale_display.load(QUrl(url)) ######################################################################################### # Signals # ######################################################################################### def _topic_selected(self, item): ''' Handler function - user selects a topic in the Topic View. Refreshes the list of documents in the Document View with all documents for which a worker judgment has been loaded for that topic. Computes statistics across that topic. ''' topic_id = item.text() # Update control selection. self._selected_topic = topic_id # Update statistics view. self.update_statistics(topic=str(topic_id)) print ("Loading documents for topic %s" % topic_id) documents = self._dm.judged_documents_by_topic(topic_id) self.update_document_list([d.id for d in documents]) def _document_selected(self, item): ''' Handler function - user select a document in the Document View. Loads the text from that document into the rationale display and computes statistics for that document. ''' document_id = item.text() # Update control selection. self._selected_document = document_id # Grab control selections. selected_topic = self._selected_topic selected_document = self._selected_document print ("Loading rationales for document %s, topic %s" % (selected_document, selected_topic)) rationales = self._dm.judgments(selected_topic, selected_document) rationales = [Rationale(str(random.randint(1,10000)), r) for r in rationales] self.update_rationale_selection(rationales) # Update statistics view. self.update_statistics(str(selected_topic), str(selected_document)) # Load document. document = next((d for d in self._dm.judged_documents() if d.id == selected_document), None) self.load_document(document.url) def _rationale_selection_changed(self, state): ''' Handler function called when a user selects or deselects a rationale check box. ''' worker = Thread(target=self.update_rationale_display) worker.start()
def FormatPlateMapData(keys_and_vals, categorical=False): ''' keys_and_vals -- a list of lists of well-keys and values eg: [['p1', 'A01', 0.2], ['p1', 'A02', 0.9], ...] returns a 2-tuple containing: -an array in the shape of the plate containing the given values with NaNs filling empty slots. If multiple sites per-well are given, then the array will be shaped (rows, cols, sites) -an array in the shape of the plate containing the given keys with (UnknownPlate, UnknownWell) filling empty slots ''' from itertools import groupby keys_and_vals = np.array(keys_and_vals) nkeycols = len(dbconnect.well_key_columns()) shape = list(p.plate_shape) if p.plate_type == '5600': well_keys = keys_and_vals[:,:-1] # first column(s) are keys data = keys_and_vals[:,-1] # last column is data assert data.ndim == 1 if len(data) < 5600: raise Exception( '''The measurement you chose to plot was missing for some spots. Because CPA doesn't know the well labelling convention used by this microarray, we can't be sure how to plot the data. If you are plotting an object measurement, you may have some spots with 0 objects and therefore no entry in the table.''') assert len(data) == 5600 data = np.array(list(meander(data.reshape(shape)))).reshape(shape) sort_indices = np.array(list(meander(np.arange(np.prod(shape)).reshape(shape)))).reshape(shape) well_keys = np.array(list(meander(well_keys.reshape(shape + [nkeycols] )))).reshape(shape + [nkeycols]) return data, well_keys, sort_indices # compute the number of sites-per-well as the max number of rows with the same well-key nsites = max([len(list(grp)) for k, grp in groupby(keys_and_vals, lambda row: tuple(row[:nkeycols])) ]) if nsites > 1: # add a sites dimension to the array shape if there's >1 site per well shape += [nsites] data = np.ones(shape) * np.nan if categorical: data = data.astype('object') if p.plate_id: dummy_key = ('UnknownPlate', 'UnknownWell') else: dummy_key = ('UnknownWell',) well_keys = np.array([dummy_key] * np.prod(shape), dtype=object).reshape(shape + [nkeycols]) sort_indices = np.ones(data.shape)*np.nan dm = DataModel.getInstance() ind = keys_and_vals.argsort(axis=0) for i, (k, well_grp) in enumerate(groupby(keys_and_vals[ind[:,len(dummy_key)-1],:], lambda row: tuple(row[:len(dummy_key)]))): (row, col) = dm.get_well_position_from_name(k[-1]) well_data = np.array(list(well_grp))[:,-1] if len(well_data) == 1: data[row, col] = well_data[0] sort_indices[row,col] = ind[:,len(dummy_key)-1][i] else: data[row, col] = well_data sort_indices[row,col] = ind[:,len(dummy_key)-1][i*nsites + np.array(range(nsites))] well_keys[row, col] = k return data, well_keys, sort_indices
def main(args): dm = DataModel() dm.read_data(to_read_count=10, normalize_data=True) dm.write_data('../data/data_imp.csv')
def captureevents(self): """ Capture BMC/IPMI System Event Log entries (SEL) and put into common DB. :param: none :returns: status, reply :raises: None Make sure IPMI is present. Get the IPMI sel logs. There may be more than 1 line (usually come in doubles). Insert into the common database. That routine eliminates duplicates. When finished, clear all SEL logs. """ # Only do this if ipmi is in the kernel and active. (Ignore VM's) if not self.ipmisupported: return 0, '' reply = "Unable to capture IPMI/BMC System Event Logs (SEL)" # Get the BMC/IPMI System Event Logs (SEL). try: # stderr=subprocess.PIPE and ignoring sel_lines.stderr -- tosses the output. sel_lines = subprocess.check_output(['/usr/bin/ipmitool', 'sel', 'list'], stderr=subprocess.PIPE) except subprocess.CalledProcessError as ex: self.logger.error(traceback.format_exc()) reply = reply + '{"exception":' + str(ex) + '}' return 1, reply # If None, leave. if sel_lines is None: return 0, '' # Convert output to usable format. sel_lines = sel_lines.decode('utf-8').splitlines() # If nothing in it, leave -- nothing new to enter into common database. if sel_lines == []: return 0, '' # Decode the SEL logs, place into format for SQL table, and put there. for a in sel_lines: # Split line into list on space pipe space. selarr = a.split(' | ') # Date and time are separated by ' | ', combine and separate with space. dt = selarr[1] + ' ' + selarr[2] # Get 'date time' in SQL format datetime. selarr[2] = parse(dt) # List slicing to remove elements 0 and 1. selarr = selarr[2:] done = False # Get events stored in common database. table_name = 'ipmievents' sqlcommand = ("SELECT * FROM {0} WHERE (datetime='{1}' and event='{2}' and details='{3}' and status='{4}')" .format(table_name, selarr[0], selarr[1], selarr[2], selarr[3])) try: ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand) except sqlalchemy.exc.OperationalError as ex: self.logger.error(traceback.format_exc()) reply = reply + '{"exception":' + str(ex) + '}' return 1, reply except Exception as ex: self.logger.error(traceback.format_exc()) reply = reply + '{"exception":' + str(ex) + '}' return 1, reply for l in ipmidata: done = True break # New entry, insert into database. if not done: query = "INSERT INTO ipmievents (datetime, event, details, status) " \ "VALUES ( '{0}', '{1}', '{2}', '{3}');" \ .format(selarr[0], selarr[1], selarr[2], selarr[3]) try: DataModel().ExecuteRawQueryStatement(query) except sqlalchemy.exc.OperationalError as ex: self.logger.error(traceback.format_exc()) reply = reply + '{"exception":' + str(ex) + '}' return 1, reply except Exception as ex: self.logger.error(traceback.format_exc()) reply = reply + '{"exception":' + str(ex) + '}' return 1, reply # Clear all SEL logs. Note: minor race condition from read to clear is possible. try: output = subprocess.check_output(['/usr/bin/ipmitool', 'sel', 'clear'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as ex: print("error in ipmitool sel clear") # DEBUG self.logger.error(traceback.format_exc()) reply = reply + '{"exception":' + str(ex) + '}' return 1, reply # Everything is good at this point. return 0, ''
def __init__(self, properties=None, parent=None, id=ID_IMAGE_GALLERY, **kwargs): if properties is not None: global p p = properties global db db = dbconnect.DBConnect.getInstance() wx.Frame.__init__(self, parent, id=id, title='CPA/ImageGallery - %s' % \ (os.path.basename(p._filename)), size=(800, 600), **kwargs) if parent is None and not sys.platform.startswith('win'): self.tbicon = wx.TaskBarIcon() self.tbicon.SetIcon(icons.get_cpa_icon(), 'CPA/ImageGallery') else: self.SetIcon(icons.get_cpa_icon()) self.SetName('ImageGallery') db.register_gui_parent(self) global dm dm = DataModel.getInstance() if not p.is_initialized(): logging.critical('ImageGallery requires a properties file. Exiting.') raise Exception('ImageGallery requires a properties file. Exiting.') self.pmb = None self.worker = None self.trainingSet = None self.classBins = [] self.binsCreated = 0 self.chMap = p.image_channel_colors[:] self.toggleChMap = p.image_channel_colors[ :] # used to store previous color mappings when toggling colors on/off with ctrl+1,2,3... self.brightness = 1.0 self.scale = 1.0 self.contrast = 'Linear' self.defaultTSFileName = None self.defaultModelFileName = None self.lastScoringFilter = None self.menuBar = wx.MenuBar() self.SetMenuBar(self.menuBar) self.CreateMenus() self.CreateStatusBar() #### Create GUI elements # Top level - three split windows self.splitter = wx.SplitterWindow(self, style=wx.NO_BORDER | wx.SP_3DSASH) self.fetch_and_rules_panel = wx.Panel(self.splitter) self.bins_splitter = wx.SplitterWindow(self.splitter, style=wx.NO_BORDER | wx.SP_3DSASH) # fetch & rules self.fetch_panel = wx.Panel(self.fetch_and_rules_panel) self.find_rules_panel = wx.Panel(self.fetch_and_rules_panel) # sorting bins self.gallery_panel = wx.Panel(self.bins_splitter) o_label = p.object_name[0] if p.classification_type == 'image' else '' + ' image gallery' self.gallery_box = wx.StaticBox(self.gallery_panel, label=o_label) self.gallery_sizer = wx.StaticBoxSizer(self.gallery_box, wx.VERTICAL) self.galleryBin = sortbin.SortBin(parent=self.gallery_panel, classifier=self, label='image gallery', parentSizer=self.gallery_sizer) self.gallery_sizer.Add(self.galleryBin, proportion=1, flag=wx.EXPAND) self.gallery_panel.SetSizer(self.gallery_sizer) self.objects_bin_panel = wx.Panel(self.bins_splitter) # fetch objects interface self.startId = wx.TextCtrl(self.fetch_panel, id=-1, value='1', size=(60, -1), style=wx.TE_PROCESS_ENTER) self.endId = wx.TextCtrl(self.fetch_panel, id=-1, value='100', size=(60, -1), style=wx.TE_PROCESS_ENTER) self.fetchChoice = wx.Choice(self.fetch_panel, id=-1, choices=['range','all','individual']) self.fetchChoice.SetSelection(0) self.filterChoice = wx.Choice(self.fetch_panel, id=-1, choices=['experiment'] + p._filters_ordered + p._groups_ordered + [ CREATE_NEW_FILTER]) self.fetchFromGroupSizer = wx.BoxSizer(wx.HORIZONTAL) self.fetchBtn = wx.Button(self.fetch_panel, -1, 'Fetch!') #### Create Sizers self.fetchSizer = wx.BoxSizer(wx.HORIZONTAL) self.find_rules_sizer = wx.BoxSizer(wx.HORIZONTAL) self.fetch_and_rules_sizer = wx.BoxSizer(wx.VERTICAL) self.classified_bins_sizer = wx.BoxSizer(wx.HORIZONTAL) #### Add elements to sizers and splitters # fetch panel self.fetchSizer.AddStretchSpacer() self.fetchSizer.Add(wx.StaticText(self.fetch_panel, -1, 'Fetch '), flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchSizer.Add(self.fetchChoice, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchTxt = wx.StaticText(self.fetch_panel, -1, label='of image IDs:') self.fetchSizer.Add(self.fetchTxt, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchSizer.Add(self.startId, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchTxt2 = wx.StaticText(self.fetch_panel, -1, label='to') self.fetchSizer.Add(self.fetchTxt2, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchSizer.Add(self.endId, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) #self.fetchSizer.Add(self.obClassChoice, flag=wx.ALIGN_CENTER_VERTICAL) #self.fetchSizer.AddSpacer((5, 20)) self.fetchTxt3 = wx.StaticText(self.fetch_panel, -1, label='images') self.fetchSizer.Add(self.fetchTxt3, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchSizer.Add(wx.StaticText(self.fetch_panel, -1, 'from'), flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchSizer.Add(self.filterChoice, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((10, 20)) self.fetchSizer.Add(self.fetchFromGroupSizer, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddSpacer((5, 20)) self.fetchSizer.Add(self.fetchBtn, flag=wx.ALIGN_CENTER_VERTICAL) self.fetchSizer.AddStretchSpacer() self.fetch_panel.SetSizerAndFit(self.fetchSizer) # fetch and rules panel self.fetch_and_rules_sizer.Add((5, 5)) self.fetch_and_rules_sizer.Add(self.fetch_panel, flag=wx.EXPAND) self.fetch_and_rules_sizer.Add((5, 5)) self.fetch_and_rules_panel.SetSizerAndFit(self.fetch_and_rules_sizer) # classified bins panel self.objects_bin_panel.SetSizer(self.classified_bins_sizer) # splitter windows self.splitter.SplitHorizontally(self.fetch_and_rules_panel, self.bins_splitter, self.fetch_and_rules_panel.GetMinSize()[1]) self.bins_splitter.SplitHorizontally(self.gallery_panel, self.objects_bin_panel) self.splitter.SetSashGravity(0.0) self.bins_splitter.SetSashGravity(0.5) self.splitter.SetMinimumPaneSize(max(50, self.fetch_and_rules_panel.GetMinHeight())) self.bins_splitter.SetMinimumPaneSize(50) self.SetMinSize((self.fetch_and_rules_panel.GetMinWidth(), 4 * 50 + self.fetch_and_rules_panel.GetMinHeight())) # Set initial state self.filterChoice.SetSelection(0) # JEN - Start Add # self.openDimensReduxBtn.Disable() # JEN - End Add self.fetchSizer.Hide(self.fetchFromGroupSizer) ##################### #### GUI Section #### ##################### # add the default classes #for class in range(1, num_classes+1): self.AddSortClass('objects of selected image') #self.AddSortClass('negative') self.Layout() self.Center() self.MapChannels(p.image_channel_colors[:]) self.BindMouseOverHelpText() #self.Bind(wx.EVT_BUTTON, self.OnInspect, self.inspectBtn) # JEN - Start Add # self.Bind(wx.EVT_BUTTON, self.OpenDimensRedux, self.openDimensReduxBtn) # JEN - End Add self.Bind(wx.EVT_BUTTON, self.OnFetch, self.fetchBtn) self.startId.Bind(wx.EVT_TEXT, self.ValidateIntegerField) self.startId.Bind(wx.EVT_TEXT_ENTER, self.OnFetch) self.Bind(wx.EVT_CLOSE, self.OnClose) self.Bind(wx.EVT_CHAR, self.OnKey) # Doesn't work for windows tilecollection.EVT_TILE_UPDATED(self, self.OnTileUpdated) self.Bind(sortbin.EVT_QUANTITY_CHANGED, self.QuantityChanged) self.Bind(wx.EVT_CHOICE, self.OnSelectFetchChoice, self.fetchChoice) self.Bind(wx.EVT_CHOICE, self.OnSelectFilter, self.filterChoice)
from sys import stderr from tempfile import gettempdir from time import ctime, time #from wx.lib.embeddedimage import PyEmbeddedImage import dbconnect import imagetools import csv import logging import numpy as np import os import sys import weakref import wx import wx.grid dm = DataModel.getInstance() db = dbconnect.DBConnect.getInstance() p = Properties.getInstance() ID_LOAD_CSV = wx.NewId() ID_SAVE_CSV = wx.NewId() ID_EXIT = wx.NewId() DO_NOT_LINK_TO_IMAGES = 'Do not link to images' ROW_LABEL_SIZE = 30 # Icon to be used for row headers (difficult to implement) #img_icon = PyEmbeddedImage('iVBORw0KGgoAAAANSUhEUgAAABUAAAASCAYAAAC0EpUuAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOR2lDQ1BQaG90b3Nob3AgSUNDIHByb2ZpbGUAAHjarZdXUNRbl8X3v5tuUpObnJqcsySRnARRckZSkyW0bYMgIiogWTKiIBlEiQoogiJJRC6CCCqgggiCERFUkNTfg3furZqqmXmZ9XDqt1edqrP2edm1AVgO+pBIoSgACAunkO3MjQgurm4E2mmgB25gBQREfIgnSIY2NlbwP+rXC0AAACYUfEikUAuqw7tL/mLqKZ9WtIcbq2fhfxcz2cXVDQCRBwB84B82AAC87x92AAD8SQqJAoAEAQCeGOTjB4CcBgB5soOdMQBSBwDMgX+4AwCYff/wEAAwRxEDKQDIFACWPdwvOByA9gsAVs/P/wQRgFkeAPz8ThDDAJizASA0LCzCD4B5BwCkiSQyBYBFHADEXVzdCH8iewOA+gAArey/XkQOQCsRgGftX0/KBIBrHaCV/l9v3Q4QAEC4xk4EqKkCAACCMwLAzFKp65IAtLkAuzlU6nYllbpbBYB+DdATSowkR/39XwjyBOD/qv/0/LfQCAAKAGQgGyEgnagotA2NKcYHm0/7ht6QoROnzdTL4sz6gT2KY48zhmuJx5K3mG9RgFNQQ8hNOIZQJNIhOi1GlRCVNJMKkc6W6ZJ9L8+qYKxIVqpRfqXKomaxL0l9QBO0TLUT9j/Q+azLprdf380g1rDc6KHxkimDmaK57cHjFrmWtw9NWW0fEbDeb+NqG2dXYt/lMOO45SzoYuIa4VbpPnkU47nPy887x6fXd81P1N8xICtwIGg3ROdYfGh3OCbCmnT1+MoJY0pB5OpJg2jfmNBT5NiY0wlxF86kx6edzTiXfD4uIToxKMnxglGyTApTypfUR2kV6ZEZlpl8mcsXW7POZjvlKOTS5L7Ku5WfUuBZqHqJ9tKbotbLqVd8irVKWEqWr/aWlpRFlztW7KvEV/6sel7dUZNR63RN7NpaXc/1zBse9XL1mw2PGouagpr3tzC0zNxsuBXXeqSN0LbW/vB2/h1ih9pd9N2nndldh+/R3Ru4f77boJv6oLvndK9uH/T19ScNWD3kePhqsObR8SHdx4yPXw03/nV2xP6JxJMfo91jKU9tx3nH3z1rmjg5qf+c4fnzF+UvQ6e0p2mnn89Uv6K8NnnD+ebjbOdc1lvivMY7pnefFvoWC977L6ksbS8//pD/kfhJ4zPT5w9fxr8Orgx+m1qlrhmtX/7J8Kt602PLbMd/L5dKBQBDWESOo7hRE+g7NC2YR9gNOl36AkYsLpLpB0scG7Cf59jjDOca45Hljea7w78kiBWSFj5I8BM5J1oh9lB8URIrJS1tJhMhmyfXIb+gyKykqeynclG1W21VXVzDXjNN6572Zx26A4K66noH9V0Mwg0TjAqNb5jcNx0zmzdfs8Ba8hySsdp/2OKIs3WIzUnbRLtc+wqHm46DTu9cUK5ibsbuPh7njpZ5dnvNeVN9RYiWfhH+lwL6A38Ei4Y4HssMHQxHRxiTzhwfIO9QOCMFooRPikVLxUifko6VOi0ZJ3qGL57jLP3ZnXMr52cTniR2JJVdSEsOS7FP1UjjTttIn8ioz0y66J6llo3LXsrpyb2SR863KVAoxBV+uTRS1HQ560pksXuJ8VW5Uq4yVNlK+UzFYGVlVVi1bg1rzUJt+7WMOu/r6jeYbryv72640khuOtQs0Uxtmb7Zciutldim1Y5rf3u77U5Kh+ddtU5s51RXw734+/bdEt0bD4Z7rvaG9xn04/uXB+4+zBz0eaQ+xDA097h9OPMvvxH9J+KjHGPosbWnb8aHnrVNlEwmPA95YfdSe0p4mnb628zUq77XzW+KZ9Pn4t5GzPu/c1+wWzz03mzJZNnwg/5H40+Wn+2+eHwNWTn1LWu17nvP2sz6xk++Xwc2vDYTftdvPdve2BXa06caUqkAwA+e0I9oIE0oPdQi+goNEaOHlaUVoOOl52MQZRTHyTPJMYuzsLNss86wNbFHcWhx/Ma3c5K5lLm+cTfzhPMq8q7ytfJHCegIIoIjQoXCPgQlwrbIiGiZGFncVEJIYl2yVypRWk96S6ZNNlxOTu6DfK1CgKKk4rJSnXKIiorKT9VOtbP7zNQZ1Sc1ijV9tKS1VrU79sfrmB1gPTCjW60Xrq9pgBiMGF4yIhorm4DJqOlVswhzk4P4gx8s7lvmHzpmZXKY//CPI4+tC2w8bMVsv9jdtj/vYOso5Ljq1Odc6BLhauYm4kZ1n/PoOVrhmeAV7G3lo+7LT0SIH/yG/esCLgT6BBkECwdTQ94e6w2tCLsQfizCmqR5XIiMJa+ceEnpi2yIKjqZGH08xuuURazKaeT0QFzCGdN4dHzP2fhzWufWzjckBCaKJr5JKrpgn8yS/CQlI9UyjTFtND07wy6TO/P1xYost2yO7L9yEnMN8yCvNz+xwKKQvXDuUlNR/GXXK2rFrMWrJZNXO0uryrLK4yoiKn2qjlSr1rDWfKp9eK2i7sx1rxv69YQGdMPHxtGm1ubSluSbpFvuraZtyu38t7G3V++87nh4t6mztKvn3mI38oC3R63Xvi+mv27g3aDso/ihL8OkEdonD8aqx9smNl7ETLu9Ln2b9/7I5+qfllQqwJ/ZBwCA1QC4jAJwGgJwpAHIaQGQygHg9AawYQJw0AYUgy4gW/OASAb8Mz84QB5MwQuiIQfq4SHMwW8EjygjVkgIkozUIoPIMooBpYhyQJ1GXUNNoNFodXQIuga9SCNOE0jTQLOO0cEkYcawAtgQbCctjvYobQsdhs6droWejt6HvpMBz3Cc4QmjLGMy4xLOHFfLRM8UyvSc2YS5nUWSpYQVz5rORsMWz7bLfop9iyOWg4pP4mTgzOHi56rhVuXu47HneccbxcfIV8m/n39CIFSQRrBESEXokbC38Bbhksg+kUlRihhe7La4m/iuxFVJI8llqXRpRekxmUhZXtkuOU95kK9UMFFYUkxXklcaVT6lYqIqqPpb7eW+VvUijVjNQC1bbeP9ujpqBzR09fUs9b0NYg0rjUZMUKbaZmfMhyzYLW0PZVtNHuG19rVptN2zt3aocNxwtnSpct11d/Ro9MR6eXnf8WUhhvr1B2gG9gYbhQyF2ofNRASTNslJFM7ImpMHosdPhZxGx5XH65ydP5+aqJr0Nvlmalb6iUy7LLUcfO52/kzh/aLSK9ElzqXK5XQVc1VtNReuOV2XqadtWGtaaJm69axt4HZfx4POu/c6utt7bvU1DzQO3hi6PlwxUjVa8rT02dXJtBfJU7kzDa+fzX6b511Qe2+0bP5R+RP1S9eK+7f574fWatdXf8r8stzw3Qz/HbTlvK2xw7ozu1u9d4lKBQAciMEBcIBwSIYK6IIX8B3BITKIGeKHnEcqkX7kPYoepYhyRMWj6lHTaHq0LpqCbkGv0qjQUGg6MSjMIUwB5i1WCXsGO0YrRhtJO0wnShdDN04vR59Mv8BgxFDBCIw+jH04KVwabpXJhamfWYu5hUWapZpVjLWCTYStml2avYlDg6MHb4Wf4Qzi/M2VwS3CfZfHnuczbyqfNN9j/mMCzAI3Be0EvwtlC6sITxJiRIRFHomSxPjEHoqTJYQkhiSjpMSlnknHycjKzMgmy6nJvZXPUNBWeK9YqGSotKrcqHJO1VNNb5+wOqL+UWNcs1urXrt8f5FO9oEc3Ty9Yv0ag3bDUaNFE4ypuJmV+cmDtRYLh4SsfA9XHVmykbM9bnffAXG0dMpxnnEVdQt3b/PY8TTxyvCe8OUl+vs1BLAHXg7Gh2SHsoVlRDCTLpJxJzIi6aOyovExxbFip9vOGMRPnCMlsCbeuuCYvJFanm6dsXOxMds7ly9vvCDj0pHLuCsjJXmlTuUCFYtVDTWUa/uvY270NRCbUM3Xbtq1otqabwd3SNyd7yq77/dAvOdD342B8EG5R98e1/1FfCI4+uxp+jODiY3ntS+dp3Ezd197z2LmGuet331dTFsiLN/6aPlp7gvp6/a3lO9cazU/lH92bJhvvtjy297dvUilAoAgmEAI5EAHzCF0iCriiaQj95AVlDjKE1WKeoeWR8ejX9Po0dRi2DHnMOvYEOwCrRftLJ033Tx9IP0KQwwjmjELR8A1MxkzzTPnspiy7LF2sFHY1dh/cNzBn+E05mLimuKu4znFa8UnwUflfyXQKVgmdF44iGAroieqICYoziaBkdiU/C71UXpO5o3sS7lJ+XGFMcVnShPKi6o0alL7Dqof00jXbNYa1V7TYT6gpHtYL1g/yaDSsM/ojfGeKZ+Zrrn3wSSLDsuvVjKHfY5csX5uy2pna5/h8NiJ3tnEJcl10J3Ow+JohudTb04fN99y4oq/UcB4EDmE5di9sIAIYdI0OZ9iF4U/OR1TEusRJ3pm9mzFeb9E8aSl5EepLenFmbFZLjnKedj80cKCIqcrtMWdVwPLmMtvVdpVfatJuSZU13bDtYGzcba57ialVbcdbj/uyOq0ucd+f+pBfq9zP8/AzGDpkP+w9F8/nnSNpY97TohO5r+gf5k6zTFT+lr8TdUc/9v0+e8LVotX3y8s83ww+Oj7KfZzwpekr+SVo980V1lWX34vW3NeZ17v+uHy49fP9F/cv6o2pDdKNlk2Eza//7b73bbFtRW9NbzNse2xXbO9sqO5E7VTu/N8F7O7b9dvN2+3f3d9T3TPeu/UXs3e071tqhTVjhpHraNOUqkAf/YlAABgMI4IjSATrIxN4P9XYaGR//UGGgBw/uGO9gDADgD7Ashmdn+zWTDFwgEA8ADgAsYQAaEQAWQggBUYgwk8BiKQwQfC/3H+nAQg/nM3+M+eBwCAZQUoOQcA0Hufeva/Z6L4R1MAAIwjSDHk4MAgCsGQRAr1lydYhBMV5Qmqysra8B9BJBMdj2+jxwAAACBjSFJNAABtmAAAc44AAONlAACDMAAAfIYAANfxAAAzGQAAGuA00fGBAAACJUlEQVR42qyUPUsjURSGnxOjSKYQSZMmTVBQQZgldVAwTa6wv2C74I+wC9jKdnamyy/YZpoIgRRbGjaIhWuRbiur+cqM49lmZnDyARvYA5fLXN773vPxviOqyroQER4eHtYDFuLi4kIAyv8CHo/Hhe8kSZjP5wRBgO/7eJ7H1dUVQA/olUWETSKKIsIwxPd9fN/HdV1c1y1gVma6WHKr1Vr7yM3NzUrSb8A5MAIGm5b8/PyMZVlLpOeq2k3bMNi05CAIeH9/XyIdiQjX19fdyWTSBbi9vS2AbNvm8vKS4XDIbDZjMpkQxzHD4ZAgCHBdN7vzFfgNcK+q2ul0dF2cnJzo2dmZNhqNwnmn01GgcAbclxZLbTabNJtNAE5PTzk+Pubp6YnZbMbr6yvGGHZ2dnK8qiIiGGMQkUdgtDR9VSVJkryXWb/m83mO+fj4KBgkM1AmzyXSOI5zoiiKiON4iTR7FODg4ABjTJbQFxE5L6+a9OfsPpPu7u4ShmEB//Ly8jnrR2BUSqfft22bw8ND9vb2sCyLWq3G9vY2pVKJSqWSyyrtXYFIRPoi0ge+A4Nyqs1Bu93uTqdTXNdFVdnf38fzPJIkQUTyPeufMQbbtnEc50fq+dU2fXt7y4WdCT0MQ6IoyjHGGBzH+QP8dBzn1+KQl/x+dHSk9Xpdq9WqVioV3draUqCw0v9Cb5XzVBVVLWZ6d3fH/wjJxJtGb4O7vZUlA38HAO/oekRA0FPwAAAAAElFTkSuQmCC') class HugeTable(wx.grid.PyGridTableBase): '''
def score_objects(properties, ts, gt, nRules, filter_name=None, group='Image', show_results=False, results_table=None, overwrite=False): ''' Trains a Classifier on a training set and scores the experiment returns the table of scores as a numpy array. properties -- Properties instance ts -- TrainingSet instance gt -- Ground Truth instance nRules -- number of rules to use filter_name -- name of a filter to use from the properties file group -- name of a group to use from the properties file show_results -- whether or not to show the results in TableViewer results_table -- table name to save results to or None. ''' p = properties #db = DBConnect.getInstance() ## Removed writing to db. Results_table should be 'None' anyway dm = DataModel.getInstance() #if group == None: #group = 'Image' if results_table: if db.table_exists(results_table) and not overwrite: print 'Table "%s" already exists. Delete this table before running scoreall.'%(results_table) return None print '' print 'properties: ', properties print 'initial training set: ', ts print 'ground truth training set: ', gt print '# rules: ', nRules print 'filter: ', filter_name print 'grouping by: ', group print 'show results: ', show_results print 'results table: ', results_table print 'overwrite: ', overwrite print '' nClasses = len(ts.labels) nKeyCols = len(image_key_columns()) assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s'%(nRules,) assert filter_name in p._filters.keys()+[None], 'Filter %s not found in properties file. Valid filters are: %s'%(filter_name, ','.join(p._filters.keys()),) assert group in p._groups.keys()+['Image', 'None'], 'Group %s not found in properties file. Valid groups are: %s'%(group, ','.join(p._groups.keys()),) output = StringIO() logging.info('Training classifier with %s rules...'%nRules) t0 = time() weaklearners = fastgentleboostingmulticlass.train(ts.colnames, nRules, ts.label_matrix, ts.values, output) logging.info('Training done in %f seconds'%(time()-t0)) t0 = time() #def update(frac): #logging.info('%d%% '%(frac*100.,)) ## Score Ground Truth using established classifier gt_predicted_scores = per_cell_scores(weaklearners, gt.values, gt.colnames) #plt.hist(gt_predicted_scores) #plt.show() gt_predicted_signs = np.sign(gt_predicted_scores) ## Compare Ground Truth score signs with the actual ground truth values numclasses = ts.labels.size gt_actual_signs = gt.label_matrix[:,0] cm_unrotated = metrics.confusion_matrix(gt_actual_signs,gt_predicted_signs) ## sklearn.metrics.confusion_matrix -- 2D confusion matrix is inverted from convention. ## https://github.com/scikit-learn/scikit-learn/issues/1664 cm = np.rot90(np.rot90(cm_unrotated)) fpr, sens, thresholds = metrics.roc_curve(gt_actual_signs,gt_predicted_signs) spec = 1-fpr s = np.sum(cm,axis=1) percent = [100*cm[i,i]/float(s[i]) for i in range(len(s))] avg = np.mean(percent) avgTotal = 100 * np.trace(cm) / float(np.sum(cm)) print 'accuracy = %f' % avgTotal print 'Confusion Matrix = ... ' print cm my_sens = cm[0,0] / float(cm[0,0] + cm[0,1]) #TP/(TP+FN) my_spec = cm[1,1] / float(cm[1,1] + cm[1,0]) #TN/(TN+FP) print 'My_Sensitivity = %f' % my_sens print 'My_Specificity = %f' % my_spec print 'Sensitivity = ...' print sens print 'Specificity = ...' print spec print 'Done calculating' ############ ## Confusion Matrix code from here: http://stackoverflow.com/questions/5821125/how-to-plot-confusion-matrix-with-string-axis-rather-than-integer-in-python conf_arr = cm norm_conf = [] ## This normalizes each *row* to the color map, but I chose to normalize the whole confusion matrix to the same scale ##for i in conf_arr: ##a = 0 ##tmp_arr = [] ##a = sum(i, 0) ##for j in i: ##tmp_arr.append(float(j)/float(a)) ##norm_conf.append(tmp_arr) norm_conf = conf_arr / float(np.max(conf_arr)) if DISPLAY_CONFUSION_MATRIX: fig = plt.figure() plt.clf() ax = fig.add_subplot(111) ax.set_aspect(1) res = ax.imshow(np.array(norm_conf), cmap=plt.cm.jet, interpolation='nearest') width = len(conf_arr) height = len(conf_arr[0]) for x in xrange(width): for y in xrange(height): ax.annotate(str(conf_arr[x][y]), xy=(y, x), horizontalalignment='center', verticalalignment='center') cb = fig.colorbar(res) #cb.set_cmap = [0,1] if width == 2 and height == 2: plt.xticks([0,1],['FP','TN']) plt.yticks([0,1],['TP','FP']) else: alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' plt.xticks(range(width), alphabet[:width]) plt.yticks(range(height), alphabet[:height]) plt.show() print 'Done'
class ParseDlg(QDialog): def __init__(self, filename, parent=None): super(ParseDlg, self).__init__(parent) self.filename = filename self.rawdata = None f = open(self.filename) data = f.read() tmp = None for delim in ['\r\n','\r','\n']: tmp = data.split(delim) l = len(tmp) if (l>1): if not tmp[l-1]: del tmp[l-1] self.rawdata = tmp break f.close() self.parent = parent self.header = None self.setModal(True) buttonBox = QDialogButtonBox() cancelButton = buttonBox.addButton(buttonBox.Cancel) QObject.connect(cancelButton, SIGNAL('clicked()'), self.close) self.okButton = buttonBox.addButton(buttonBox.Ok) QObject.connect(self.okButton, SIGNAL('clicked()'), self.doParse) mainLayout = QGridLayout() importBox = QGroupBox('Import') #importBox.setSizePolicy(QSizePolicy.Maximum,QSizePolicy.Maximum) importBoxLayout = QGridLayout() importBoxLayout.addWidget(QLabel('Character set'),0,0) self.charsetComboBox = QComboBox() self.charsetComboBox.addItems(['utf-8']) importBoxLayout.addWidget(self.charsetComboBox,0,1) importBox.setLayout(importBoxLayout) separatorBox = QGroupBox('Separator Options') #separatorBox.setSizePolicy(QSizePolicy.Maximum,QSizePolicy.Maximum) separatorBoxLayout = QGridLayout() separatorBoxLayout.setHorizontalSpacing(60) self.delimTab = QRadioButton('Tab') self.delimOther = QRadioButton('Other') self.otherLineEdit = QLineEdit(',') self.otherLineEdit.setMaxLength(1) separatorBoxLayout.addWidget(self.delimTab,0,0) separatorBoxLayout.addWidget(self.delimOther,0,2) separatorBoxLayout.addWidget(self.otherLineEdit,0,3) separatorBox.setLayout(separatorBoxLayout) otherBox = QGroupBox('Other Options') #otherBox.setSizePolicy(QSizePolicy.Maximum,QSizePolicy.Maximum) otherBoxLayout = QGridLayout() otherBoxLayout.setHorizontalSpacing(60) self.quotedFieldCheckBox = QCheckBox('Quoted field as text') self.textLabel = QLabel('Quote character') self.textLabel.setAlignment(Qt.AlignRight|Qt.AlignVCenter) self.textLineEdit = QLineEdit('"') self.textLineEdit.setMaxLength(1) self.headerCheckBox = QCheckBox('First row as header') self.skipCommentsCheckBox = QCheckBox('Skip comment lines') self.commentLabel = QLabel('Comment prefix') self.commentLabel.setAlignment(Qt.AlignRight|Qt.AlignVCenter) self.commentLineEdit = QLineEdit('#') self.stripCheckBox = QCheckBox('Strip lines') self.regexLabel = QLabel('Regex') self.regexLabel.setAlignment(Qt.AlignRight|Qt.AlignVCenter) self.stripLineEdit = QLineEdit('^\(|\)$') otherBoxLayout.addWidget(self.quotedFieldCheckBox,0,0) otherBoxLayout.addWidget(self.textLabel,0,1) otherBoxLayout.addWidget(self.textLineEdit,0,2) otherBoxLayout.addWidget(self.headerCheckBox,1,0) otherBoxLayout.addWidget(self.skipCommentsCheckBox,2,0) otherBoxLayout.addWidget(self.commentLabel,2,1) otherBoxLayout.addWidget(self.commentLineEdit,2,2) otherBoxLayout.addWidget(self.stripCheckBox,3,0) otherBoxLayout.addWidget(self.regexLabel,3,1) otherBoxLayout.addWidget(self.stripLineEdit,3,2) otherBox.setLayout(otherBoxLayout) filterBox = QGroupBox('Sample Filter') filterBoxLayout = QGridLayout() filterBoxLayout.setHorizontalSpacing(60) filterBoxLayout.addWidget(QLabel('Column'),0,0) self.columnFilterComboBox = QComboBox() filterBoxLayout.addWidget(self.columnFilterComboBox,1,0) filterBoxLayout.addWidget(QLabel('Sample Indicator'),2,0) self.sampleIndicatorComboBox = QComboBox() self.sampleIndicatorComboBox.setEnabled(False) filterBoxLayout.addWidget(self.sampleIndicatorComboBox,3,0) QObject.connect(self.columnFilterComboBox, SIGNAL('currentIndexChanged(int)'), self.filterColChanged) QObject.connect(self.sampleIndicatorComboBox, SIGNAL('currentIndexChanged(int)'), self.sampleIndicatorChanged) filterBox.setLayout(filterBoxLayout) columnBox = QGroupBox('Column Options') columnBoxLayout = QGridLayout() columnBoxLayout.setHorizontalSpacing(60) columnBoxLayout.addWidget(QLabel('Status'),0,0) self.statusCombo = QComboBox() columnBoxLayout.addWidget(self.statusCombo,0,1,1,2) self.comparisonCombo = QComboBox() self.comparisonCombo.addItem('==',operator.eq) self.comparisonCombo.addItem('<',operator.lt) self.comparisonCombo.addItem('<=',operator.le) self.comparisonCombo.addItem('>',operator.gt) self.comparisonCombo.addItem('>=',operator.ge) columnBoxLayout.addWidget(self.comparisonCombo,0,3) self.comparisonValue = QLineEdit('1') columnBoxLayout.addWidget(self.comparisonValue,0,4,1,2) columnBoxLayout.addWidget(QLabel('Gaze X'),1,0) self.gazexCombo = QComboBox() columnBoxLayout.addWidget(self.gazexCombo,1,1,1,2) columnBoxLayout.addWidget(QLabel('Gaze Y'),2,0) self.gazeyCombo = QComboBox() columnBoxLayout.addWidget(self.gazeyCombo,2,1,1,2) columnBoxLayout.addWidget(QLabel('Timestamp'),1,3) self.timestampCombo = QComboBox() columnBoxLayout.addWidget(self.timestampCombo,1,4,1,2) columnBoxLayout.addWidget(QLabel('Trial'),2,3) self.trialCombo = QComboBox() columnBoxLayout.addWidget(self.trialCombo,2,4,1,2) columnBox.setLayout(columnBoxLayout) previewBox = QGroupBox('Preview') previewBoxLayout = QGridLayout() self.datatableModel = None self.datatable = QTableView() previewBoxLayout.addWidget(self.datatable) previewBox.setLayout(previewBoxLayout) mainLayout.addWidget(importBox,0,0) mainLayout.addWidget(separatorBox,0,1) mainLayout.addWidget(otherBox,1,1) mainLayout.addWidget(filterBox,1,0) mainLayout.addWidget(columnBox,3,0,1,2) mainLayout.addWidget(previewBox,4,0,1,2) mainLayout.addWidget(buttonBox,5,0,1,2) self.delimTab.setChecked(True) QObject.connect(self.headerCheckBox, SIGNAL('stateChanged(int)'), self.refreshDataTable) QObject.connect(self.delimTab, SIGNAL('toggled(bool)'), self.updatePreview) QObject.connect(self.delimOther, SIGNAL('toggled(bool)'), self.updatePreview) QObject.connect(self.otherLineEdit, SIGNAL('textChanged(const QString&)'), self.otherDelimChanged) QObject.connect(self.textLineEdit, SIGNAL('textChanged(const QString&)'), self.textChanged) QObject.connect(self.quotedFieldCheckBox, SIGNAL('stateChanged(int)'), self.updatePreview) QObject.connect(self.skipCommentsCheckBox, SIGNAL('stateChanged(int)'), self.updatePreview) QObject.connect(self.stripCheckBox, SIGNAL('stateChanged(int)'), self.updatePreview) self.skipCommentsCheckBox.setCheckState(Qt.CheckState.Checked) self.setLayout(mainLayout) self.setWindowTitle("Data Import - [%s]" % (self.filename)) self.show() #self.setFixedSize(self.width(),self.height()) self.updatePreview() def sampleIndicatorChanged(self, index): self.updatePreview() def filterColChanged(self, index): if index>0: index -= 1 self.sampleIndicatorComboBox.setEnabled(True) items = [] data = self.parseData(tmp=True) if self.headerCheckBox.isChecked(): data = data[1:] for x in data: try: items.append(x[index]) except IndexError: pass items = list(frozenset(items)) items.insert(0,'') self.sampleIndicatorComboBox.clear() self.sampleIndicatorComboBox.addItems(items) else: self.sampleIndicatorComboBox.clear() self.sampleIndicatorComboBox.setEnabled(False) def textChanged(self): if len(self.textLineEdit.text()) == 1: self.updatePreview() def otherDelimChanged(self): if len(self.otherLineEdit.text()) == 1: self.updatePreview() def parseData(self, preview=False, progress_cb=None, tmp=False): reg0 = None if self.skipCommentsCheckBox.isChecked() and self.commentLineEdit.text(): reg0 = re.compile('^'+self.commentLineEdit.text()) reg1 = None if self.stripCheckBox.isChecked() and self.stripLineEdit.text(): reg1 = re.compile(self.stripLineEdit.text()) lines = [] l = len(self.rawdata) for i, line in enumerate(self.rawdata): if progress_cb: progress_cb(i+1,l) line = line.encode(self.charsetComboBox.itemText(self.charsetComboBox.currentIndex())) if reg0 and reg0.match(line): continue if reg1: line = reg1.sub('', line) lines.append(line) if preview and len(lines) == 100: break quotechar = None if self.quotedFieldCheckBox.isChecked() and self.textLineEdit.text(): quotechar = str(self.textLineEdit.text()) delim = None if self.delimTab.isChecked(): delim = '\t' elif self.delimOther.isChecked() and self.otherLineEdit.text(): delim = str(self.otherLineEdit.text()) tmpData = None if not tmp and self.columnFilterComboBox.currentIndex() > 0 and self.sampleIndicatorComboBox.currentIndex() > 0: col = self.columnFilterComboBox.itemData(self.columnFilterComboBox.currentIndex()) val = self.sampleIndicatorComboBox.itemText(self.sampleIndicatorComboBox.currentIndex()) if self.headerCheckBox.isChecked(): tmpData = [line for i,line in enumerate(csv.reader(lines, delimiter=delim, quotechar=quotechar)) if i==0 or line[col] == val] else: tmpData = [line for line in csv.reader(lines, delimiter=delim, quotechar=quotechar) if line[col] == val] else: tmpData = [line for line in csv.reader(lines, delimiter=delim, quotechar=quotechar)] if not tmp: self.data = tmpData return tmpData def updatePreview(self): self.parseData(preview=True) self.refreshDataTable() def doParse(self): self.done(1) def updateModel(self): if self.headerCheckBox.isChecked(): self.datatableModel = DataModel(self.data, True) else: self.datatableModel = DataModel(self.data, False) def refreshDataTable(self): if not self.data: return self.updateModel() self.datatable.setModel(self.datatableModel) self.datatable.horizontalHeader().setVisible(True) self.datatable.verticalHeader().setVisible(True) self.datatable.setShowGrid(True) self.datatable.resizeColumnsToContents() self.datatable.resizeRowsToContents() headers = copy.copy(self.datatableModel.getHeader()) headers.insert(0,None) if headers != self.header: self.updateHeaderCombo(self.statusCombo, headers) self.updateHeaderCombo(self.gazexCombo, headers) self.updateHeaderCombo(self.gazeyCombo, headers) self.updateHeaderCombo(self.timestampCombo, headers) self.updateHeaderCombo(self.trialCombo, headers) self.updateHeaderCombo(self.columnFilterComboBox, headers) self.header = headers def updateHeaderCombo(self, combobox, headers): combobox.clear() for i,v in enumerate(headers): if i == 0: combobox.addItem(v,None) else: combobox.addItem(v,i-1) def getSegments(self, status_cb=None): segments = [] tidx = self.trialCombo.currentIndex()-1 sidx = self.statusCombo.currentIndex()-1 data = self.datatableModel.getData() header = self.datatableModel.getHeader() current = [header] status = [] line = 0 lines = len(data) trial = 1 if tidx != -1: trial = data[line][tidx] while line < lines: if tidx != -1 and data[line][tidx] != trial: segments.append({'trial': trial, 'data': current, 'status': status}) current = [header] status = [] trial = data[line][tidx] if sidx == -1: status.append(True) else: status.append(self.comparisonCombo.itemData(self.comparisonCombo.currentIndex())(float(data[line][sidx]),float(self.comparisonValue.text()))) current.append(data[line]) line += 1 segments.append({'trial': trial, 'data': current, 'status': status}) return {self.filename: {'segments': segments, 'firstRowIsHeader': self.headerCheckBox.isChecked()}}
import numpy as np from datamodel import DataModel if __name__=='__main__': if False: from tetrahedral import ChiralTetrahedral as Group ## from tetrahedral import Null as Group ## from octahedral import Tetrahedral as Group ## from octahedral import Null as Group from octahedral import Pyritohedral as Group ## from octahedral import ChiralOctahedral as Group ## from octahedral import Tetrahedral as Group ## from octahedral import Origin as Group ## from dihedral import ChiralDihedral as Group ## from icosahedral import ChiralIcosahedral as Group ## from icosahedral import Icosahedral as Group dm = DataModel(Group()) else: ## dm = DataModel.load(r'C:\Users\Eelco\Dropbox\Escheresque\examples\fishes.sch') ## dm = DataModel.load(r'C:\Users\Eelco\Dropbox\Escheresque\examples\angles_and_demons.sch') dm = DataModel.load(r'C:\Users\Eelco Hoogendoorn\Dropbox\Escheresque\code\v2\test.sch') print dm.edges
def updateModel(self): if self.headerCheckBox.isChecked(): self.datatableModel = DataModel(self.data, True) else: self.datatableModel = DataModel(self.data, False)
def score(properties, ts, nRules, filter_name=None, group='Image', show_results=False, results_table=None, overwrite=False): ''' Trains a Classifier on a training set and scores the experiment returns the table of scores as a numpy array. properties -- Properties instance ts -- TrainingSet instance nRules -- number of rules to use filter_name -- name of a filter to use from the properties file group -- name of a group to use from the properties file show_results -- whether or not to show the results in TableViewer results_table -- table name to save results to or None. ''' p = properties db = DBConnect.getInstance() dm = DataModel.getInstance() if group == None: group = 'Image' if results_table: if db.table_exists(results_table) and not overwrite: print 'Table "%s" already exists. Delete this table before running scoreall.' % ( results_table) return None print '' print 'properties: ', properties print 'training set: ', ts print '# rules: ', nRules print 'filter: ', filter_name print 'grouping by: ', group print 'show results: ', show_results print 'results table: ', results_table print 'overwrite: ', overwrite print '' nClasses = len(ts.labels) nKeyCols = len(image_key_columns()) assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s' % ( nRules, ) assert filter_name in p._filters.keys() + [ None ], 'Filter %s not found in properties file. Valid filters are: %s' % ( filter_name, ','.join(p._filters.keys()), ) assert group in p._groups.keys() + [ 'Image' ], 'Group %s not found in properties file. Valid groups are: %s' % ( group, ','.join(p._groups.keys()), ) output = StringIO() logging.info('Training classifier with %s rules...' % nRules) t0 = time() weaklearners = fastgentleboostingmulticlass.train(ts.colnames, nRules, ts.label_matrix, ts.values, output) logging.info('Training done in %f seconds' % (time() - t0)) logging.info('Computing per-image class counts...') t0 = time() def update(frac): logging.info('%d%% ' % (frac * 100., )) keysAndCounts = multiclasssql.PerImageCounts(weaklearners, filter_name=(filter_name or None), cb=update) keysAndCounts.sort() logging.info('Counts found in %f seconds' % (time() - t0)) if not keysAndCounts: logging.error( 'No images are in filter "%s". Please check the filter definition in your properties file.' % (filter_name)) raise Exception( 'No images are in filter "%s". Please check the filter definition in your properties file.' % (filter_name)) # AGGREGATE PER_IMAGE COUNTS TO GROUPS IF NOT GROUPING BY IMAGE if group != 'Image': logging.info('Grouping %s counts by %s...' % (p.object_name[0], group)) t0 = time() imData = {} for row in keysAndCounts: key = tuple(row[:nKeyCols]) imData[key] = np.array([float(v) for v in row[nKeyCols:]]) groupedKeysAndCounts = np.array([ list(k) + vals.tolist() for k, vals in dm.SumToGroup(imData, group).items() ], dtype=object) nKeyCols = len(dm.GetGroupColumnNames(group)) logging.info('Grouping done in %f seconds' % (time() - t0)) else: groupedKeysAndCounts = np.array(keysAndCounts, dtype=object) # FIT THE BETA BINOMIAL logging.info('Fitting beta binomial distribution to data...') counts = groupedKeysAndCounts[:, -nClasses:] alpha, converged = polyafit.fit_betabinom_minka_alternating(counts) logging.info(' alpha = %s converged = %s' % (alpha, converged)) logging.info(' alpha/Sum(alpha) = %s' % ([a / sum(alpha) for a in alpha])) # CONSTRUCT ARRAY OF TABLE DATA logging.info('Computing enrichment scores for each group...') t0 = time() tableData = [] for i, row in enumerate(groupedKeysAndCounts): # Start this row with the group key: tableRow = list(row[:nKeyCols]) if group != 'Image': tableRow += [ len(dm.GetImagesInGroup(group, tuple(row[:nKeyCols]))) ] # Append the counts: countsRow = [int(v) for v in row[nKeyCols:nKeyCols + nClasses]] tableRow += [sum(countsRow)] tableRow += countsRow if p.area_scoring_column is not None: # Append the areas countsRow = [int(v) for v in row[-nClasses:]] tableRow += [sum(countsRow)] tableRow += countsRow # Append the scores: # compute enrichment probabilities of each class for this image OR group scores = np.array(dirichletintegrate.score(alpha, np.array(countsRow))) # clamp to [0,1] to scores[scores > 1.] = 1. scores[scores < 0.] = 0. tableRow += scores.tolist() # Append the logit scores: # Special case: only calculate logit of "positives" for 2-classes if nClasses == 2: tableRow += [np.log10(scores[0]) - (np.log10(1 - scores[0])) ] # compute logit of each probability else: tableRow += [ np.log10(score) - (np.log10(1 - score)) for score in scores ] # compute logit of each probability tableData.append(tableRow) tableData = np.array(tableData, dtype=object) logging.info('Enrichments computed in %f seconds' % (time() - t0)) # CREATE COLUMN LABELS LIST # if grouping isn't per-image, then get the group key column names. if group != 'Image': colnames = dm.GetGroupColumnNames(group) else: colnames = list(image_key_columns()) # record the column indices for the keys key_col_indices = [i for i in range(len(colnames))] if group != 'Image': colnames += ['Number_of_Images'] colnames += ['Total_%s_Count' % (p.object_name[0].capitalize())] for i in xrange(nClasses): colnames += [ '%s_%s_Count' % (ts.labels[i].capitalize(), p.object_name[0].capitalize()) ] if p.area_scoring_column is not None: colnames += ['Total_%s_Area' % (p.object_name[0].capitalize())] for i in xrange(nClasses): colnames += [ '%s_%s_Area' % (ts.labels[i].capitalize(), p.object_name[0].capitalize()) ] for i in xrange(nClasses): colnames += ['pEnriched_%s' % (ts.labels[i])] if nClasses == 2: colnames += ['Enriched_Score_%s' % (ts.labels[0])] else: for i in xrange(nClasses): colnames += ['Enriched_Score_%s' % (ts.labels[i])] title = results_table or "Enrichments_per_%s" % (group, ) if filter_name: title += "_filtered_by_%s" % (filter_name, ) title += ' (%s)' % (os.path.split(p._filename)[1]) if results_table: print 'Creating table %s' % (results_table) success = db.CreateTableFromData(tableData, colnames, results_table, temporary=False) if not success: print 'Failed to create results table :(' if show_results: import tableviewer tableview = tableviewer.TableViewer(None, title=title) if results_table and overwrite: tableview.load_db_table(results_table) else: tableview.table_from_array(tableData, colnames, group, key_col_indices) tableview.set_fitted_col_widths() tableview.Show() return tableData
def score(properties, ts, nRules, filter_name=None, group='Image', show_results=False, results_table=None, overwrite=False): ''' Trains a Classifier on a training set and scores the experiment returns the table of scores as a numpy array. properties -- Properties instance ts -- TrainingSet instance nRules -- number of rules to use filter_name -- name of a filter to use from the properties file group -- name of a group to use from the properties file show_results -- whether or not to show the results in TableViewer results_table -- table name to save results to or None. ''' p = properties db = DBConnect.getInstance() dm = DataModel.getInstance() if group == None: group = 'Image' if results_table: if db.table_exists(results_table) and not overwrite: print 'Table "%s" already exists. Delete this table before running scoreall.'%(results_table) return None print '' print 'properties: ', properties print 'training set: ', ts print '# rules: ', nRules print 'filter: ', filter_name print 'grouping by: ', group print 'show results: ', show_results print 'results table: ', results_table print 'overwrite: ', overwrite print '' nClasses = len(ts.labels) nKeyCols = len(image_key_columns()) assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s'%(nRules,) assert filter_name in p._filters.keys()+[None], 'Filter %s not found in properties file. Valid filters are: %s'%(filter_name, ','.join(p._filters.keys()),) assert group in p._groups.keys()+['Image'], 'Group %s not found in properties file. Valid groups are: %s'%(group, ','.join(p._groups.keys()),) output = StringIO() logging.info('Training classifier with %s rules...'%nRules) t0 = time() weaklearners = fastgentleboostingmulticlass.train(ts.colnames, nRules, ts.label_matrix, ts.values, output) logging.info('Training done in %f seconds'%(time()-t0)) logging.info('Computing per-image class counts...') t0 = time() def update(frac): logging.info('%d%% '%(frac*100.,)) keysAndCounts = multiclasssql.PerImageCounts(weaklearners, filter_name=(filter_name or None), cb=update) keysAndCounts.sort() logging.info('Counts found in %f seconds'%(time()-t0)) if not keysAndCounts: logging.error('No images are in filter "%s". Please check the filter definition in your properties file.'%(filter_name)) raise Exception('No images are in filter "%s". Please check the filter definition in your properties file.'%(filter_name)) # AGGREGATE PER_IMAGE COUNTS TO GROUPS IF NOT GROUPING BY IMAGE if group != 'Image': logging.info('Grouping %s counts by %s...' % (p.object_name[0], group)) t0 = time() imData = {} for row in keysAndCounts: key = tuple(row[:nKeyCols]) imData[key] = np.array([float(v) for v in row[nKeyCols:]]) groupedKeysAndCounts = np.array([list(k)+vals.tolist() for k, vals in dm.SumToGroup(imData, group).items()], dtype=object) nKeyCols = len(dm.GetGroupColumnNames(group)) logging.info('Grouping done in %f seconds'%(time()-t0)) else: groupedKeysAndCounts = np.array(keysAndCounts, dtype=object) # FIT THE BETA BINOMIAL logging.info('Fitting beta binomial distribution to data...') counts = groupedKeysAndCounts[:,-nClasses:] alpha, converged = polyafit.fit_betabinom_minka_alternating(counts) logging.info(' alpha = %s converged = %s'%(alpha, converged)) logging.info(' alpha/Sum(alpha) = %s'%([a/sum(alpha) for a in alpha])) # CONSTRUCT ARRAY OF TABLE DATA logging.info('Computing enrichment scores for each group...') t0 = time() tableData = [] for i, row in enumerate(groupedKeysAndCounts): # Start this row with the group key: tableRow = list(row[:nKeyCols]) if group != 'Image': tableRow += [len(dm.GetImagesInGroup(group, tuple(row[:nKeyCols])))] # Append the counts: countsRow = [int(v) for v in row[nKeyCols:nKeyCols+nClasses]] tableRow += [sum(countsRow)] tableRow += countsRow if p.area_scoring_column is not None: # Append the areas countsRow = [int(v) for v in row[-nClasses:]] tableRow += [sum(countsRow)] tableRow += countsRow # Append the scores: # compute enrichment probabilities of each class for this image OR group scores = np.array( dirichletintegrate.score(alpha, np.array(countsRow)) ) # clamp to [0,1] to scores[scores>1.] = 1. scores[scores<0.] = 0. tableRow += scores.tolist() # Append the logit scores: # Special case: only calculate logit of "positives" for 2-classes if nClasses==2: tableRow += [np.log10(scores[0])-(np.log10(1-scores[0]))] # compute logit of each probability else: tableRow += [np.log10(score)-(np.log10(1-score)) for score in scores] # compute logit of each probability tableData.append(tableRow) tableData = np.array(tableData, dtype=object) logging.info('Enrichments computed in %f seconds'%(time()-t0)) # CREATE COLUMN LABELS LIST # if grouping isn't per-image, then get the group key column names. if group != 'Image': colnames = dm.GetGroupColumnNames(group) else: colnames = list(image_key_columns()) # record the column indices for the keys key_col_indices = [i for i in range(len(colnames))] if group != 'Image': colnames += ['Number_of_Images'] colnames += ['Total_%s_Count'%(p.object_name[0].capitalize())] for i in xrange(nClasses): colnames += ['%s_%s_Count'%(ts.labels[i].capitalize(), p.object_name[0].capitalize())] if p.area_scoring_column is not None: colnames += ['Total_%s_Area'%(p.object_name[0].capitalize())] for i in xrange(nClasses): colnames += ['%s_%s_Area'%(ts.labels[i].capitalize(), p.object_name[0].capitalize())] for i in xrange(nClasses): colnames += ['pEnriched_%s'%(ts.labels[i])] if nClasses==2: colnames += ['Enriched_Score_%s'%(ts.labels[0])] else: for i in xrange(nClasses): colnames += ['Enriched_Score_%s'%(ts.labels[i])] title = results_table or "Enrichments_per_%s"%(group,) if filter_name: title += "_filtered_by_%s"%(filter_name,) title += ' (%s)'%(os.path.split(p._filename)[1]) if results_table: print 'Creating table %s'%(results_table) success = db.CreateTableFromData(tableData, colnames, results_table, temporary=False) if not success: print 'Failed to create results table :(' if show_results: import tableviewer tableview = tableviewer.TableViewer(None, title=title) if results_table and overwrite: tableview.load_db_table(results_table) else: tableview.table_from_array(tableData, colnames, group, key_col_indices) tableview.set_fitted_col_widths() tableview.Show() return tableData
def score_objects(properties, ts, gt, nRules, filter_name=None, group='Image', show_results=False, results_table=None, overwrite=False): ''' Trains a Classifier on a training set and scores the experiment returns the table of scores as a numpy array. properties -- Properties instance ts -- TrainingSet instance gt -- Ground Truth instance nRules -- number of rules to use filter_name -- name of a filter to use from the properties file group -- name of a group to use from the properties file show_results -- whether or not to show the results in TableViewer results_table -- table name to save results to or None. ''' p = properties #db = DBConnect.getInstance() ## Removed writing to db. Results_table should be 'None' anyway dm = DataModel.getInstance() #if group == None: #group = 'Image' if results_table: if db.table_exists(results_table) and not overwrite: print 'Table "%s" already exists. Delete this table before running scoreall.' % ( results_table) return None print '' print 'properties: ', properties print 'initial training set: ', ts print 'ground truth training set: ', gt print '# rules: ', nRules print 'filter: ', filter_name print 'grouping by: ', group print 'show results: ', show_results print 'results table: ', results_table print 'overwrite: ', overwrite print '' nClasses = len(ts.labels) nKeyCols = len(image_key_columns()) assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s' % ( nRules, ) assert filter_name in p._filters.keys() + [ None ], 'Filter %s not found in properties file. Valid filters are: %s' % ( filter_name, ','.join(p._filters.keys()), ) assert group in p._groups.keys() + [ 'Image', 'None' ], 'Group %s not found in properties file. Valid groups are: %s' % ( group, ','.join(p._groups.keys()), ) output = StringIO() logging.info('Training classifier with %s rules...' % nRules) t0 = time() weaklearners = fastgentleboostingmulticlass.train(ts.colnames, nRules, ts.label_matrix, ts.values, output) logging.info('Training done in %f seconds' % (time() - t0)) t0 = time() #def update(frac): #logging.info('%d%% '%(frac*100.,)) ## Score Ground Truth using established classifier gt_predicted_scores = per_cell_scores(weaklearners, gt.values, gt.colnames) #plt.hist(gt_predicted_scores) #plt.show() gt_predicted_signs = np.sign(gt_predicted_scores) ## Compare Ground Truth score signs with the actual ground truth values numclasses = ts.labels.size gt_actual_signs = gt.label_matrix[:, 0] cm_unrotated = metrics.confusion_matrix(gt_actual_signs, gt_predicted_signs) ## sklearn.metrics.confusion_matrix -- 2D confusion matrix is inverted from convention. ## https://github.com/scikit-learn/scikit-learn/issues/1664 cm = np.rot90(np.rot90(cm_unrotated)) fpr, sens, thresholds = metrics.roc_curve(gt_actual_signs, gt_predicted_signs) spec = 1 - fpr s = np.sum(cm, axis=1) percent = [100 * cm[i, i] / float(s[i]) for i in range(len(s))] avg = np.mean(percent) avgTotal = 100 * np.trace(cm) / float(np.sum(cm)) print 'accuracy = %f' % avgTotal print 'Confusion Matrix = ... ' print cm my_sens = cm[0, 0] / float(cm[0, 0] + cm[0, 1]) #TP/(TP+FN) my_spec = cm[1, 1] / float(cm[1, 1] + cm[1, 0]) #TN/(TN+FP) print 'My_Sensitivity = %f' % my_sens print 'My_Specificity = %f' % my_spec print 'Sensitivity = ...' print sens print 'Specificity = ...' print spec print 'Done calculating' ############ ## Confusion Matrix code from here: http://stackoverflow.com/questions/5821125/how-to-plot-confusion-matrix-with-string-axis-rather-than-integer-in-python conf_arr = cm norm_conf = [] ## This normalizes each *row* to the color map, but I chose to normalize the whole confusion matrix to the same scale ##for i in conf_arr: ##a = 0 ##tmp_arr = [] ##a = sum(i, 0) ##for j in i: ##tmp_arr.append(float(j)/float(a)) ##norm_conf.append(tmp_arr) norm_conf = conf_arr / float(np.max(conf_arr)) if DISPLAY_CONFUSION_MATRIX: fig = plt.figure() plt.clf() ax = fig.add_subplot(111) ax.set_aspect(1) res = ax.imshow(np.array(norm_conf), cmap=plt.cm.jet, interpolation='nearest') width = len(conf_arr) height = len(conf_arr[0]) for x in xrange(width): for y in xrange(height): ax.annotate(str(conf_arr[x][y]), xy=(y, x), horizontalalignment='center', verticalalignment='center') cb = fig.colorbar(res) #cb.set_cmap = [0,1] if width == 2 and height == 2: plt.xticks([0, 1], ['FP', 'TN']) plt.yticks([0, 1], ['TP', 'FP']) else: alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' plt.xticks(range(width), alphabet[:width]) plt.yticks(range(height), alphabet[:height]) plt.show() print 'Done'
# -*- coding: utf-8 -*- from PyQt4.QtGui import QApplication import os import sys from mainWindow import MainWindow from datamodel import DataModel from moses import Moses if __name__ == "__main__": app = QApplication(sys.argv) workdir = os.path.join(os.path.join(os.path.expanduser('~'), 'mosesgui')) if not os.path.exists(workdir): os.makedirs(workdir) dm = DataModel(filename=os.path.join(workdir, "models.sqlite")) moses = Moses() if not moses.detect(): sys.exit(1) MainWindow = MainWindow(dm=dm, moses=moses, workdir=workdir) MainWindow.setupUi(MainWindow) MainWindow.show() sys.exit(app.exec_())