def test_data_size_vs_diff(dm, given_dict, infer_dict):
#Read all data from data model
dm.read_data(normalize_data=False)
#attr_list = [U_UNIVERSITY_CODE, PROGRAM_CODE, UNIVERSITY, MAJOR_CODE, TERM]
attr_list = [U_UNIVERSITY_CODE, PROGRAM_CODE, UNIVERSITY]
#attr_list = [MAJOR_CODE, PROGRAM_CODE, TERM]
#Size of data
data_size = len(dm.data)
#Step size = 10 steps
step_size = data_size//10
#Get experiment data in a dict
size = []
accuracy = []
for i in xrange(step_size, data_size, step_size):
dm_test = DataModel("")
dm_test.set_data(dm.data[:i])
exp_test = Experimenter(dm_test, attr_list)
actual = exp_test.get_actual_result(given_dict, infer_dict)
estimation = exp_test.generic_get_estimated_result(given_dict, infer_dict)
size.append(i)
accuracy.append(abs(estimation - actual))
print("Step:%d--->Actual:%f--->Estimate:%f" %(i, actual, estimation))
print "-------------------------------------------------------------"
plt.figure()
plt.plot(size, accuracy)
plt.title("Data Size vs Accuracy")
plt.show()
def train_lstm(model_filename, weights_filename, l1_d=128, l2_d=128, b_s=128, bi=False):
"""
Trains a 2 layer lstm and saves the model in the files specified.
Args:
model_filename: The filename to save the model
weights_filename: The filename to save the weights
l1_d: Layer one dimensions
l2_d: Layer two dimensions
b_s: Batch size
Returns:
model: The lstm model fit on the training data
"""
dm=DataModel()
texts, labels=dm.get_train_data()
word_index, data=utils.get_word_index(texts)
x_train, y_train, x_val, y_val, _, _=utils.get_train_val_test_data(
data, labels)
word_embeddings=utils.get_glove_embeddings()
embedding_matrix=utils.get_embedding_matrix(word_embeddings, word_index)
if bi:
model=_bi_lstm(embedding_matrix, x_train, y_train, x_val,
y_val, 3, word_index, l1_d, l2_d, b_s)
else:
model=_lstm(embedding_matrix, x_train, y_train, x_val,
y_val, 3, word_index, l1_d, l2_d, b_s)
save_model(model, model_filename, weights_filename)
return model
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.setupUi(self)
self.mplWidget = MplWidget.warp_a_widget(self.matplotlibWidget)
self.model = DataModel(self.mplWidget.canvas)
self.tableView.setModel(self.model)
validator = QDoubleValidator(0, 1000, 5)
self.leastIntLineEdit.setValidator(validator)
self.maxZDiffLineEdit.setValidator(validator)
self.leastIntLineEdit.setText('200')
self.maxZDiffLineEdit.setText('1')
# Signals-Slots
self.actionImportData.triggered.connect(self.setSampleFile)
self.startFindButton.clicked.connect(self.startSearch)
def setSampleFile(self):
filename, _ = QFileDialog.getOpenFileName(
self, "Input File", "", "Text files (*.txt);;All File(*)")
if not filename:
return
self.model.setSample(filename)
def startSearch(self):
self.model.maxDiff = float(self.maxZDiffLineEdit.text())
self.model.minInt = float(self.leastIntLineEdit.text())
self.model.search()
def run_lstm_on_test_data(model_filename, weights_filename, kaggle_filename):
dm = DataModel()
test_x, test_id = dm.get_test_data()
print(test_x.shape)
_, data = utils.get_word_index(test_x)
# word_embeddings = utils.get_glove_embeddings()
# _ = utils.get_embedding_matrix(word_embeddings, word_index)
y = utils.load_model_and_evaluate(model_filename, weights_filename, data)
utils.write_kaggle_file(y, test_id, kaggle_filename)
def main(args):
dm = DataModel(args.gig_file, args.chat_file)
dm.read_data()
exp = Experimenter(dm)
if args.classify is True:
scores = exp.classify_gigs()
if args.feature_values is True:
scores = exp.evaluate_feature_values()
return dm
def main(argv):
train_count = -1
if(len(argv)>0):
train_count = int(argv[0])
dm = DataModel()
dm.get_data(train_count)
# (training, feature_names) = get_rich_featured_training(dm,lines)
print(len(dm.data.keys()))
print(len(dm.train))
print(len(dm.test))
def main(argv):
train_count = -1
if (len(argv) > 0):
train_count = int(argv[0])
dm = DataModel()
dm.get_data(train_count)
# (training, feature_names) = get_rich_featured_training(dm,lines)
print(len(dm.data.keys()))
print(len(dm.train))
print(len(dm.test))
def main(args):
dm = DataModel(args.train_file)
dm.read_train_data()
exp = Experimenter(dm)
distances = [x.get_distance() for x in dm.data]
print(max(distances))
print(min(distances))
print(stats.mean(distances))
t1 = time.time()
t2 = time.time()
timeused = t2 - t1
logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \
(timeused/3600, timeused/60, timeused%60))
return exp
def accuracy_test(model_filename, weights_filename):
dm = DataModel()
x_train, y_train = dm.get_train_data()
train_samples = int(0.8 * len(x_train))
x_test = x_train[train_samples:]
y_test = y_train[train_samples:]
_, data = utils.get_word_index(x_test)
labels = to_categorical(np.asarray(y_test))
# word_embeddings = utils.get_glove_embeddings()
# _ = utils.get_embedding_matrix(word_embeddings, word_index)
md = utils.load_model(model_filename, weights_filename)
result = md.evaluate(data, labels)
print('\nTest loss:', result[0])
print('Test accuracy:', result[1])
def listevents(self, outputfile):
"""
Get common DB entries for table ipmievents, stored by captureevents.
:param: outputfile - JSON output file, if None - special format to stdout.
:returns: status, reply
:raises: None
"""
table_name = 'ipmievents'
if outputfile is not None:
reply = "{'" + table_name + "':["
else:
reply = ''
sqlcommand = "SELECT id, datetime, event, details, status from {0}".format(table_name)
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
except Exception as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
# Print out the ipmievents data in the common database.
first = True
ipmidata_keys = ipmidata.keys()
for entry in ipmidata:
if outputfile is not None:
reply = reply + ("\n {" if first else "},\n {")
first = second = False
for v in ipmidata_keys:
reply = ((reply + ",") if second else reply) + "'" + str(v) + "':'" + str(entry[v]) + "'"
second = True
self.logger.info(entry)
else:
print(entry[0], '|', entry[1], '|', entry[2], '|', entry[3], '|', entry[4])
if outputfile is not None:
if ipmidata is not None:
reply = reply + '}'
if outputfile is not None:
reply = reply + "\n]}"
self.logger.info("status=0 json={0}".format(reply))
return 0, reply
def __init__(self, parent=None):
super().__init__(parent)
self.setupUi(self)
self.mplWidget = MplWidget.warp_a_widget(self.matplotlibWidget)
self.model = DataModel(self.mplWidget.canvas)
self.tableView.setModel(self.model)
validator = QDoubleValidator(0, 1000, 5)
self.leastIntLineEdit.setValidator(validator)
self.maxZDiffLineEdit.setValidator(validator)
self.leastIntLineEdit.setText('200')
self.maxZDiffLineEdit.setText('1')
# Signals-Slots
self.actionImportData.triggered.connect(self.setSampleFile)
self.startFindButton.clicked.connect(self.startSearch)
def deleteevents(self, ids=None, outputfile=None):
status = 0
table_name = 'ipmievents'
# delet all event records if no ids provided
sqlcommand = "DELETE from {0}".format(table_name)
# delete selected event records
if ids is not None:
sqlcommand += " WHERE id IN ({0})".format(ids)
if outputfile is not None:
reply = "{'" + table_name + "':["
else:
reply = ''
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
status = 1
except Exception as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
status = 1
if outputfile is not None:
reply = reply + "\n]}"
self.logger.info("status=0 json={0}".format(reply))
return status, reply
def perform_datasize_vs_efficiency(self, given_dict, infer_dict, max_datasize=None, steps=10):
sizes, est_times, acc_times = [], [], []
if max_datasize is None:
max_datasize = len(self.dm.data)
data_step = max_datasize / steps
for i in range(steps):
cur_datasize = (i+1) * data_step
data = self.dm.data
while len(data) < cur_datasize:
data.extend(self.dm.data)
cur_data = data[:cur_datasize]
cur_dm = DataModel("")
cur_dm.set_data(cur_data)
cur_exp = Experimenter(cur_dm, self.attr_list)
(cur_est, cur_acc) = cur_exp.time_n_queries(given_dict, infer_dict)
sizes.append(cur_datasize)
est_times.append(float(sum(cur_est))/len(cur_est))
acc_times.append(float(sum(cur_acc))/len(cur_acc))
return (sizes, est_times, acc_times)
def generate_w2c_word_embeddings():
"""
Generates word 2 vector embeddings.
"""
dm = DataModel()
train_x, _ = dm.get_train_data()
w2c = Word2Vec(method="skipgram",
corpus=list(train_x),
window_size=5,
n_hidden=128,
n_epochs=3,
learning_rate=0.08)
W1, W2, loss_vs_epoch = w2c.run()
pkl_dump = [W1, W2, loss_vs_epoch]
with open('embeddings.pickle', 'wb') as handle:
pickle.dump(pkl_dump, handle)
def OnOpenImage(self, evt=None):
# 1) Get the image key
# Start with the table_id if there is one
tblNum = None
if p.table_id:
dlg = wx.TextEntryDialog(self, p.table_id + ':',
'Enter ' + p.table_id)
dlg.SetValue('0')
if dlg.ShowModal() == wx.ID_OK:
try:
tblNum = int(dlg.GetValue())
except ValueError:
errdlg = wx.MessageDialog(
self, 'Invalid value for %s!' % (p.table_id),
"Invalid value", wx.OK | wx.ICON_EXCLAMATION)
errdlg.ShowModal()
return
dlg.Destroy()
else:
dlg.Destroy()
return
# Then get the image_id
dlg = wx.TextEntryDialog(self, p.image_id + ':', 'Enter ' + p.image_id)
dlg.SetValue('')
if dlg.ShowModal() == wx.ID_OK:
try:
imgNum = int(dlg.GetValue())
except ValueError:
errdlg = wx.MessageDialog(
self, 'Invalid value for %s!' % (p.image_id),
"Invalid value", wx.OK | wx.ICON_EXCLAMATION)
errdlg.ShowModal()
return
dlg.Destroy()
else:
dlg.Destroy()
return
# Build the imkey
if p.table_id:
imkey = (tblNum, imgNum)
else:
imkey = (imgNum, )
dm = DataModel.getInstance()
if imkey not in dm.GetAllImageKeys():
errdlg = wx.MessageDialog(self, 'There is no image with that key.',
"Couldn't find image",
wx.OK | wx.ICON_EXCLAMATION)
errdlg.ShowModal()
self.Destroy()
else:
# load the image
self.img_key = imkey
self.SetImage(imagetools.FetchImage(imkey), p.image_channel_colors)
self.DoLayout()
def export(items, tempdir):
"""Export a list of items
Arguments
items -- list of items to export
tempdir -- directory to use for the export operation
"""
initdir = spm.spmanager.getFirstPath([spm.ExportFolder,
spm.ImportFolder,
spm.MostRecentFolder])
filenamepath = tkFileDialog.asksaveasfilename(initialdir = initdir,
filetypes = ff.dlgExportFormats,
defaultextension = ff.dlgDefaultExportExt)
if(len(filenamepath) < 1):
return
spm.spmanager.setPath(spm.ExportFolder, os.path.dirname(filenamepath))
#Create export dir and datamodel
dmdir = os.path.join(tempdir, _exportdir)
if(os.path.exists(dmdir)):
shutil.rmtree(dmdir)
os.makedirs(dmdir)
dm = DataModel(dmdir)
#Add all slideshows
for item in items:
if(not dm.addSlideshow(item, True)):
showerror(lang[lng.txtExportError], lang[lng.txtCouldNotExport] + item.title)
shutil.rmtree(dmdir)
return
#Save and zip
dm.saveToFile()
pack(dmdir, filenamepath)
shutil.rmtree(dmdir)
def main(args):
dm = DataModel(args.data_file)
dm.read_data(to_read_count=10000)
exp = Experimenter(dm, \
process_datamodel=True, \
serialise=False)
t1 = time.time()
exp.perform_multiclass_experiment(
pred_mode=INDEPENDENT,
use_exclusion=True,
need_to_extract_features=True,
prediction_file='../results/predictions_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.csv',
result_file='../results/results_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.txt',
english_only=True,
legible_only=True)
t2 = time.time()
timeused = t2 - t1
logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \
(timeused/3600, timeused/60, timeused%60))
return exp
def OnOpenImage(self, evt=None):
# 1) Get the image key
# Start with the table_id if there is one
tblNum = None
if p.table_id:
dlg = wx.TextEntryDialog(self, p.table_id + ":", "Enter " + p.table_id)
dlg.SetValue("0")
if dlg.ShowModal() == wx.ID_OK:
try:
tblNum = int(dlg.GetValue())
except ValueError:
errdlg = wx.MessageDialog(
self, "Invalid value for %s!" % (p.table_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION
)
errdlg.ShowModal()
return
dlg.Destroy()
else:
dlg.Destroy()
return
# Then get the image_id
dlg = wx.TextEntryDialog(self, p.image_id + ":", "Enter " + p.image_id)
dlg.SetValue("")
if dlg.ShowModal() == wx.ID_OK:
try:
imgNum = int(dlg.GetValue())
except ValueError:
errdlg = wx.MessageDialog(
self, "Invalid value for %s!" % (p.image_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION
)
errdlg.ShowModal()
return
dlg.Destroy()
else:
dlg.Destroy()
return
# Build the imkey
if p.table_id:
imkey = (tblNum, imgNum)
else:
imkey = (imgNum,)
dm = DataModel.getInstance()
if imkey not in dm.GetAllImageKeys():
errdlg = wx.MessageDialog(
self, "There is no image with that key.", "Couldn't find image", wx.OK | wx.ICON_EXCLAMATION
)
errdlg.ShowModal()
self.Destroy()
else:
# load the image
self.img_key = imkey
self.SetImage(imagetools.FetchImage(imkey), p.image_channel_colors)
self.DoLayout()
def updateTargetHBAs(self):
self.logger.info("ENTERED updateTargetHBAs")
try:
db = DataModel()
table = 'target_hbas'
self.logger.info("clearing table: {0}".format(table))
cmd = 'DELETE FROM {0} WHERE nodeId={1}'.format(table, self.nodeID)
db.ExecuteRawQueryStatement(cmd)
for wwn in self.__get_target_hba_wwns():
cmd = ("INSERT INTO {0} (wwn, created_at, updated_at, nodeId) VALUES ('{1}',"
"(select datetime('now')), (select datetime('now')), {2});".format(table, wwn, self.nodeID))
self.logger.info("Executing the following command: {0}".format(cmd))
db.ExecuteRawQueryStatement(cmd)
self.logger.info("Exiting updateTargetHBAs")
return True
except Exception as ex:
self.logger.error("An exception occurred while adding new wwn to TargetHBA table: {0}".format(ex))
return False
def main(args):
dm = DataModel(args.data_file)
dm.read_data(to_read_count=10000)
exp = Experimenter(dm, \
process_datamodel=True, \
serialise=False)
t1 = time.time()
exp.perform_multiclass_experiment(
pred_mode=INDEPENDENT,
use_exclusion=True,
need_to_extract_features=True,
prediction_file=
'../results/predictions_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.csv',
result_file=
'../results/results_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.txt',
english_only=True,
legible_only=True)
t2 = time.time()
timeused = t2 - t1
logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \
(timeused/3600, timeused/60, timeused%60))
return exp
def perform_datasize_vs_accuracy(self, given_dict, infer_dict, max_datasize=None, steps=10):
#Get experiment data in a dict
size = []
accuracy = []
if max_datasize is None:
max_datasize = len(self.dm.data)
data_step = max_datasize / steps
for i in range(steps):
cur_datasize = (i+1) * data_step
data = self.dm.data
while len(data) < cur_datasize:
data.extend(self.dm.data)
cur_data = data[:cur_datasize]
cur_dm = DataModel("")
cur_dm.set_data(cur_data)
cur_exp = Experimenter(cur_dm, self.attr_list)
actual = cur_exp.get_actual_result(given_dict, infer_dict)
estimation = cur_exp.generic_get_estimated_result(given_dict, infer_dict)
size.append(cur_datasize)
accuracy.append(abs(estimation - actual))
return (size, accuracy)
def listhbas():
"""
\b
NAME
listhbas - will show a list of hbas
SYNOPSIS
listhbas [OPTIONS]
DESCRIPTION
Will list out available hbas seen by the LightSpeed
No options are available for this command
"""
targetHBAStable = DataModel()
cmd = "SELECT id,wwn from target_hbas"
q = targetHBAStable.GetListOfQuery(cmd)
cli_logger.info('List of available target HBAs: ')
cli_logger.info(
'---------------------------------------------------------------------------'
)
for row in q:
cli_logger.info('ID: {0} | HBA: {1}'.format(row[0], row[1]))
cli_logger.info(
'---------------------------------------------------------------------------'
)
def get_image_keys_at_row(self, row):
'''Returns a list of image keys at the given row or None if the column
names can't be found in col_labels
'''
if self.key_indices is None or self.grouping is None:
return None
else:
if self.grouping.lower() == 'image':
return [tuple(self.data[self.row_order,:][row, self.key_indices])]
elif self.grouping.lower() == 'object':
return [tuple(self.data[self.row_order,:][row, self.key_indices[:-1]])]
else:
dm = DataModel.getInstance()
return dm.GetImagesInGroup(self.grouping, self.get_row_key(row))
def PerImageCounts(self, filter_name=None, cb=None):
# Clear the current perClassObjects storage
for bin in self.classBins:
self.perClassObjects[bin.label] = []
# Retrieve a data model instance
dm = DataModel.getInstance()
# Retrieve image keys and initialize variables
imageKeys = dm.GetAllImageKeys(filter_name)
imageAmount = float(len(imageKeys))
perImageData = []
# Process all images
for k_index, imKey in enumerate(imageKeys):
try:
# Retrieve the keys of the objects in the current image
obKeys = dm.GetObjectsFromImage(imKey)
except:
raise 'No such image: %s' % (imKey, )
return
# Calculate the amount of hits for each of the classes in the current image
classHits = {}
objectCount = [imKey[0]]
if obKeys:
classObjects = self.FilterObjectsFromClassN(keys=[imKey])
for clNum, bin in enumerate(self.classBins):
# Get the objects from the image which belong to the selected class
classHits[bin.label] = classObjects[float(clNum + 1)]
# Store the total object count of this class for the current image
nrHits = len(classHits[bin.label])
objectCount.append(nrHits)
# Store the objects for the current class and image grouped
# by class if any are found for this class in the selected image
if nrHits > 0:
self.perClassObjects[bin.label] += classHits[bin.label]
else:
# If there are objects in the image, add zeros for all bins
[objectCount.append(0) for bin in self.classBins]
# Store the results for the current image and update the callback
# function if available
perImageData.append(objectCount)
if cb:
cb(min(1, k_index / imageAmount))
return perImageData
def PerImageCounts(self, filter_name=None, cb=None):
# Clear the current perClassObjects storage
for bin in self.classBins:
self.perClassObjects[bin.label] = []
# Retrieve a data model instance
dm = DataModel.getInstance()
# Retrieve image keys and initialize variables
imageKeys = dm.GetAllImageKeys(filter_name)
imageAmount = float(len(imageKeys))
perImageData = []
# Process all images
for k_index, imKey in enumerate(imageKeys):
try:
# Retrieve the keys of the objects in the current image
obKeys = dm.GetObjectsFromImage(imKey)
except:
raise ValueError('No such image: %s' % (imKey,))
# Calculate the amount of hits for each of the classes in the current image
classHits = {}
objectCount = [imKey[0]]
if obKeys:
classObjects = self.FilterObjectsFromClassN(keys = [imKey])
for clNum, bin in enumerate(self.classBins):
# Get the objects from the image which belong to the selected class
classHits[bin.label] = classObjects[float(clNum+1)]
# Store the total object count of this class for the current image
nrHits = len(classHits[bin.label])
objectCount.append(nrHits)
# Store the objects for the current class and image grouped
# by class if any are found for this class in the selected image
if nrHits > 0:
self.perClassObjects[bin.label] += classHits[bin.label]
else:
# If there are objects in the image, add zeros for all bins
[objectCount.append(0) for bin in self.classBins]
# Store the results for the current image and update the callback
# function if available
perImageData.append(objectCount)
if cb:
cb(min(1, k_index/imageAmount))
return perImageData
def updateInitiatorHBAs(self):
# This function will update all HBA related information. There are two types present:
# a) Target HBA's
# QLE which will operate in Target mode. Target HBA's will need to be configured in database
# and LIO config.
# b) Initiator HBA's database entries only. (Some could be QLE.)
self.logger.info("ENTERED updateInitiatorHBAs")
try:
table = 'initiator_hbas'
db = DataModel()
self.logger.info("clearing table: {0}".format(table))
cmd = 'DELETE FROM {0} WHERE nodeId={1}'.format(table, self.nodeID)
db.ExecuteRawQueryStatement(cmd)
self.logger.info("Now adding the provided wwns.")
for wwn in self.__get_initiator_hba_wwns():
self.logger.info("Adding the wwn {0} to the table".format(wwn))
cmd = ("INSERT INTO {0} (wwn, created_at, updated_at, nodeId) VALUES ('{1}', (select datetime('now')), "
"(select datetime('now')), {2});".format(table, wwn, self.nodeID))
self.logger.info("Executing the command: {0}".format(cmd))
db.ExecuteRawQueryStatement(cmd)
return True
except Exception as ex:
self.logger.error("An exception occurred while updating the initiator table: {0}".format(ex))
return False
def sel_timeupdated(self):
# Update the timeupdated table with domain ipmievents for datetime now.
domain_name = 'ipmievents'
table_name = 'timeupdated'
sqlcommand = ("UPDATE {0} SET datetime=DATETIME('now') WHERE domain='{1}'"
.format(table_name, domain_name))
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
print('{"exception":' + str(ex) + '}')
return 1
except Exception as ex:
self.logger.error(traceback.format_exc())
print('{"exception":' + str(ex) + '}')
return 1
return 0
def get_object_keys_at_row(self, row):
'''Returns a list of object keys at the given row or None if the column
names can't be found in col_labels
'''
if self.key_indices is None or self.grouping is None:
return None
else:
dm = DataModel.getInstance()
# If the key index for the row is an object key, just return that key
if self.grouping.lower() == 'object':
return [tuple(self.data[self.row_order,:][row, self.key_indices])]
else: # Otherwise, return all object keys in the image
imkeys = self.get_image_keys_at_row(row)
obkeys = []
for imkey in imkeys:
obs = dm.GetObjectCountFromImage(imkey)
obkeys += [tuple(list(imkey)+[i]) for i in range(1,obs+1)]
return obkeys
def on_dclick_label(self, evt):
'''Handle display of images and objects'''
if evt.Row >= 0:
obkeys = self.grid.Table.get_object_keys_at_row(evt.Row)
if self.grid.Table.grouping is None:
# We need to know how the table is grouped to know what to do
logging.warn(
'CPA does not know how to link this table to your images. Can\'t launch ImageViewer.'
)
return
elif self.grid.Table.grouping.lower() == 'object':
# For per-object grouping, show the objects in the image
imview = imagetools.ShowImage(obkeys[0][:-1],
p.image_channel_colors,
parent=self.Parent)
if obkeys is not None:
for obkey in obkeys:
imview.SelectObject(obkey)
elif self.grid.Table.grouping.lower() == 'image':
# For per-image grouping just show the images.
# If there is only one object, then highlight it
if obkeys is not None and len(obkeys) == 1:
imview = imagetools.ShowImage(obkeys[0][:-1],
p.image_channel_colors,
parent=self.Parent)
imview.SelectObject(obkeys[0])
else:
imkeys = self.grid.Table.get_image_keys_at_row(evt.Row)
if imkeys:
#XXX: warn if there are a lot
for imkey in imkeys:
imagetools.ShowImage(imkey,
p.image_channel_colors,
parent=self.Parent)
else:
key_cols = self.grid.Table.get_row_key(evt.Row)
if key_cols:
dm = DataModel.getInstance()
imkeys = dm.GetImagesInGroup(self.grid.Table.grouping,
key_cols)
for imkey in imkeys:
imagetools.ShowImage(imkey,
p.image_channel_colors,
parent=self.Parent)
def __init__(self):
self._dm = DataModel() # Primary data model.
self._topics = [] # All topics for which judgments have been loaded.
self._documents = [] # Documents for which we have a judgment for the
# currently selected topic.
self._selected_topic = None # Currently selected topic.
self._selected_document = None # Currently selected document.
self._rationales = [] # Rationales for the currently selected document.
self._display_text = None # Text of document being manipulated.
super(CWR, self).__init__()
self.init_UI()
# For testing WebView.
'''
def get_image_keys_at_row(self, row):
'''Returns a list of image keys at the given row or None if the column
names can't be found in col_labels
'''
if self.key_indices is None or self.grouping is None:
return None
else:
if self.grouping.lower() == 'image':
return [
tuple(self.data[self.row_order, :][row, self.key_indices])
]
elif self.grouping.lower() == 'object':
return [
tuple(self.data[self.row_order, :][row,
self.key_indices[:-1]])
]
else:
dm = DataModel.getInstance()
return dm.GetImagesInGroup(self.grouping,
self.get_row_key(row))
def on_dclick_label(self, evt):
'''Handle display of images and objects'''
if evt.Row >= 0:
obkeys = self.grid.Table.get_object_keys_at_row(evt.Row)
if self.grid.Table.grouping is None:
# We need to know how the table is grouped to know what to do
logging.warn('CPA does not know how to link this table to your images. Can\'t launch ImageViewer.')
return
elif self.grid.Table.grouping.lower() == 'object':
# For per-object grouping, show the objects in the image
imview = imagetools.ShowImage(obkeys[0][:-1],
p.image_channel_colors,
parent=self.Parent)
if obkeys is not None:
for obkey in obkeys:
imview.SelectObject(obkey)
elif self.grid.Table.grouping.lower() == 'image':
# For per-image grouping just show the images.
# If there is only one object, then highlight it
if obkeys is not None and len(obkeys) == 1:
imview = imagetools.ShowImage(obkeys[0][:-1],
p.image_channel_colors,
parent=self.Parent)
imview.SelectObject(obkeys[0])
else:
imkeys = self.grid.Table.get_image_keys_at_row(evt.Row)
if imkeys:
#XXX: warn if there are a lot
for imkey in imkeys:
imagetools.ShowImage(imkey, p.image_channel_colors,
parent=self.Parent)
else:
key_cols = self.grid.Table.get_row_key(evt.Row)
if key_cols:
dm = DataModel.getInstance()
imkeys = dm.GetImagesInGroup(self.grid.Table.grouping, key_cols)
for imkey in imkeys:
imagetools.ShowImage(imkey, p.image_channel_colors,
parent=self.Parent)
def get_object_keys_at_row(self, row):
'''Returns a list of object keys at the given row or None if the column
names can't be found in col_labels
'''
if self.key_indices is None or self.grouping is None:
return None
else:
dm = DataModel.getInstance()
# If the key index for the row is an object key, just return that key
if self.grouping.lower() == 'object':
return [
tuple(self.data[self.row_order, :][row, self.key_indices])
]
else: # Otherwise, return all object keys in the image
imkeys = self.get_image_keys_at_row(row)
obkeys = []
for imkey in imkeys:
obs = dm.GetObjectCountFromImage(imkey)
obkeys += [
tuple(list(imkey) + [i]) for i in range(1, obs + 1)
]
return obkeys
def FilterObjectsFromClassN(self, classN = None, keys = None):
'''
Filter the input objects to output the keys of those in classN,
using a defined SVM model classifier.
'''
# Retrieve instance of the database connection
db = dbconnect.DBConnect.getInstance()
object_data = {}
if isinstance(keys, str):
object_data[0] = db.GetCellDataForClassifier(keys)
elif keys != []:
if len(keys) == len(dbconnect.image_key_columns()):
# Retrieve instance of the data model and retrieve objects in the requested image
dm = DataModel.getInstance()
obKeys = dm.GetObjectsFromImage(keys[0])
else:
obKeys = keys
for key in obKeys:
object_data[key] = db.GetCellDataForClassifier(key)
sorted_keys = sorted(object_data.keys())
values_array = np.array([object_data[key] for key in sorted_keys])
scaled_values = self.ScaleData(values_array)
pred_labels = self.model.predict(scaled_values)
# Group the object keys per class
classObjects = {}
for index in range(1, len(self.classBins)+1):
classObjects[float(index)] = []
for index, label in enumerate(pred_labels):
classObjects[np.int(label)+1].append(sorted_keys[index])
# Return either a summary of all classes and their corresponding objects
# or just the objects for a specific class
if classN is None:
return classObjects
else:
return classObjects[classN]
def FilterObjectsFromClassN(self, classN=None, keys=None):
'''
Filter the input objects to output the keys of those in classN,
using a defined SVM model classifier.
'''
# Retrieve instance of the database connection
db = dbconnect.DBConnect.getInstance()
object_data = {}
if isinstance(keys, str):
object_data[0] = db.GetCellDataForClassifier(keys)
elif keys != []:
if len(keys) == len(dbconnect.image_key_columns()):
# Retrieve instance of the data model and retrieve objects in the requested image
dm = DataModel.getInstance()
obKeys = dm.GetObjectsFromImage(keys[0])
else:
obKeys = keys
for key in obKeys:
object_data[key] = db.GetCellDataForClassifier(key)
sorted_keys = sorted(object_data.keys())
values_array = np.array([object_data[key] for key in sorted_keys])
scaled_values = self.ScaleData(values_array)
pred_labels = self.model.predict(scaled_values)
# Group the object keys per class
classObjects = {}
for index in range(1, len(self.classBins) + 1):
classObjects[float(index)] = []
for index, label in enumerate(pred_labels):
classObjects[np.int(label) + 1].append(sorted_keys[index])
# Return either a summary of all classes and their corresponding objects
# or just the objects for a specific class
if classN is None:
return classObjects
else:
return classObjects[classN]
def getconfiguration(self):
"""
Fetch IPMI configuration from database
:param: none
:returns: status, reply
:raises: None
"""
status = 1
obj = PxJSON("Unable to obtain IPMI configuration")
try:
table_name = 'systemsetups'
res = DataModel().ExecuteRawQueryStatement("SELECT ipmi_connection_type, ipmi_address, ipmi_netmask, ipmi_gateway, ipmi_vlan from {0}".format(table_name))
reply = {}
for row in res:
self.logger.info(row)
reply[PxJSON.CONNECTIONTYPE] = row['ipmi_connection_type']
reply[PxJSON.IPV4] = row['ipmi_address']
reply[PxJSON.NETMASK] = row['ipmi_netmask']
reply[PxJSON.GATEWAY] = row['ipmi_gateway']
reply[PxJSON.VLAN] = row['ipmi_vlan'] if row['ipmi_vlan'] != 0 else 'undefined'
obj.setroute(PxJSON.IPMI_INFO, reply)
obj.setsuccess()
status = 0
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
obj.internal({"exception": str(ex)})
except Exception as ex:
self.logger.error(traceback.format_exc())
obj.internal({"exception": str(ex)})
self.logger.info("status={0} json={1}".format(status, obj.getjsonpretty()))
return status, obj.getjson()
def get_object_keys_at_row(self, row):
# XXX: needs to be updated to work for per_well data
if self.table_name == p.image_table:
# return all objects in this image
key = self.get_row_key(row)
if key is None:
return None
dm = DataModel.getInstance()
n_objects = dm.GetObjectCountFromImage(key)
return [tuple(list(key) + [i]) for i in range(n_objects)]
elif self.table_name == p.object_table:
key = self.get_row_key(row)
if key is None:
return None
return [key]
else:
key = []
for col in dbconnect.object_key_columns():
if col not in self.col_labels:
return None
else:
col_index = self.col_labels.tolist().index(col)
key += [self.GetValue(row, col_index)]
return [tuple(key)]
def get_object_keys_at_row(self, row):
# XXX: needs to be updated to work for per_well data
if self.table_name == p.image_table:
# return all objects in this image
key = self.get_row_key(row)
if key is None:
return None
dm = DataModel.getInstance()
n_objects = dm.GetObjectCountFromImage(key)
return [tuple(list(key) + [i]) for i in range(n_objects)]
elif self.table_name == p.object_table:
key = self.get_row_key(row)
if key is None:
return None
return [key]
else:
key = []
for col in dbconnect.object_key_columns():
if col not in self.col_labels:
return None
else:
col_index = self.col_labels.tolist().index(col)
key += [self.GetValue(row, col_index)]
return [tuple(key)]
self.tile_collection.cv.acquire()
heappush(self.tile_collection.loadq, ((0, 0, 0), '<ABORT>'))
self.tile_collection.cv.notify()
self.tile_collection.cv.release()
################# FOR TESTING ##########################
if __name__ == "__main__":
app = wx.PySimpleApp()
from datamodel import DataModel
p = Properties.getInstance()
p.LoadFile('../properties/nirht_test.properties')
db = DBConnect.getInstance()
db.connect()
dm = DataModel.getInstance()
test = TileCollection.getInstance()
f = wx.Frame(None)
for i in xrange(10):
obKey = dm.GetRandomObject()
test.GetTileData((0, 1, 1), f)
for t in threading.enumerate():
if t != threading.currentThread():
t.abort()
f.Destroy()
app.MainLoop()
class CWR(QtGui.QWidget):
def __init__(self):
self._dm = DataModel() # Primary data model.
self._topics = [] # All topics for which judgments have been loaded.
self._documents = [] # Documents for which we have a judgment for the
# currently selected topic.
self._selected_topic = None # Currently selected topic.
self._selected_document = None # Currently selected document.
self._rationales = [] # Rationales for the currently selected document.
self._display_text = None # Text of document being manipulated.
super(CWR, self).__init__()
self.init_UI()
# For testing WebView.
'''
test_url = 'https://en.wikipedia.org/wiki/The_Beatles'
content = requests.get(test_url).text
content
test_text = BeautifulSoup(content, "html.parser").get_text()
self._display_text = test_text
'''
def init_UI(self):
grid = QtGui.QGridLayout()
grid.setSpacing(10)
#####################################
# Summary of UI Elements #
#####################################
# These are UI elements that are updated after creation.
self._confusion_matrix = None # String form of confusion matrix for current view.
#####################################
# Topic View #
#####################################
# Contains Topic List.
topic_view = QtGui.QGroupBox("Topics")
topic_layout = QtGui.QVBoxLayout()
topic_view.setLayout(topic_layout)
grid.addWidget(topic_view, 0, 0)
# Topic List
self._topic_list = QtGui.QListWidget()
self._topic_list.setSizePolicy(QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Minimum)
self._topic_list.itemClicked.connect(self._topic_selected)
topic_layout.addWidget(self._topic_list)
#####################################
# Document View #
#####################################
# Contains Document List.
document_view = QtGui.QGroupBox("Documents")
document_layout = QtGui.QVBoxLayout()
document_view.setLayout(document_layout)
grid.addWidget(document_view, 0, 1)
# Document List
self._document_list = QtGui.QListWidget()
self._document_list.setSizePolicy(QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
self._document_list.itemClicked.connect(self._document_selected)
document_layout.addWidget(self._document_list)
#####################################
# Statistics View #
#####################################
# Below document view. Contains the confusion matrix, list of
# rationales, gold standard values, and user judgments.
#stat_label = QtGui.QLabel()
#stat_label.setText("<B>Statistics</B>")
#stat_layout = QtGui.QVBoxLayout()
#document_layout.addWidget(stat_label)
#self._stat_display.setLayout(stat_layout)
#self._stat_display.setSizePolicy(QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Minimum)
# Topic Display
topic = QtGui.QLabel("<b>Topic</b>: N/A")
document_layout.addWidget(topic)
narrative_label = QtGui.QLabel("<b>Narrative</b:")
document_layout.addWidget(narrative_label)
narrative_view = QtGui.QTextEdit()
document_layout.addWidget(narrative_view)
# Confusion Matrix for current Topic or Topic-Document Pair
confusion_matrix_label = QtGui.QLabel()
confusion_matrix_label.setText("Confusion Matrix")
document_layout.addWidget(confusion_matrix_label)
confusion_matrix = QtGui.QLabel()
confusion_matrix.setText(" - - - - \n"
" - - - - \n"
" - - - - \n")
document_layout.addWidget(confusion_matrix)
# Gold Standard for current Topic-Document.
gold_standard_view = QtGui.QLabel()
gold_standard_view.setText("Gold Standard: N/A")
document_layout.addWidget(gold_standard_view)
# Degree 1 Agreement
d1_agreement_view = QtGui.QLabel()
d1_agreement_view.setText("D1 Agreement: N/A")
document_layout.addWidget(d1_agreement_view)
# Degree 2 Agreement
d2_agreement_view = QtGui.QLabel()
d1_agreement_view.setText("D2 Agreement: N/A")
document_layout.addWidget(d2_agreement_view)
# Give pointers to updateable elements.
self._topic_view = topic
self._narrative_view = narrative_view
self._confusion_matrix = confusion_matrix
self._gold_standard_view = gold_standard_view
self._d1_agreement_view = d1_agreement_view
self._d2_agreement_view = d2_agreement_view
#####################################
# Rationale View #
#####################################
# Contains the rationale check boxes, text display, and statistics.
rationale_view = QtGui.QGridLayout()
grid.addLayout(rationale_view, 0, 2)
# Rationale Display
display = HighlightWebView()
display.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
display.load(QUrl('https://en.wikipedia.org/wiki/The_Beatles'))
display.show()
self._rationale_display = display
rationale_view.addWidget(self._rationale_display, 2, 0)
# Rationale Selection
selection_view = QtGui.QGroupBox()
self._selection_layout = QtGui.QHBoxLayout()
selection_view.setLayout(self._selection_layout)
rationale_view.addWidget(selection_view, 1, 0)
#####################################
# Worker View #
#####################################
# Below rationale view. Contains pure text of worker rationales,
# judgments, and the gold standard.
self._worker_display = QtGui.QGroupBox("Workers")
worker_layout = QtGui.QVBoxLayout()
grid.addLayout(worker_layout, 0, 3)
#self._worker_display.setLayout(worker_layout)
# Worker Rationales for current Topic-Document.
worker_rationale_list_label = QtGui.QLabel()
worker_rationale_list_label.setText("Worker Rationales")
worker_layout.addWidget(worker_rationale_list_label)
worker_rationale_list = QtGui.QTextEdit()
worker_rationale_list.setText("Initial Text.")
worker_layout.addWidget(worker_rationale_list)
# Worker Judgments for current Topic-Document.
worker_judgment_list_label = QtGui.QLabel()
worker_judgment_list_label.setText("Worker Judgments")
worker_layout.addWidget(worker_judgment_list_label)
worker_judgment_list = QtGui.QTextEdit()
worker_judgment_list.setText("Initial Text.")
worker_layout.addWidget(worker_judgment_list)
# Give pointers to updateable elements.
self._worker_judgments = worker_judgment_list
self._worker_rationales = worker_rationale_list
#####################################
# Main Application Properties #
#####################################
self.setLayout(grid)
self.setGeometry(300, 300, 1000, 1000)
self.setWindowTitle("The Crowdworker's Rationale")
self.show()
def load(self, directory):
'''
Loads rationale data from the specified file.
'''
dm = self._dm
for dirpath, _, filenames in os.walk(directory):
for f in filenames:
if f.endswith(".csv"):
absolute_path = os.path.abspath(os.path.join(dirpath, f))
dm.load(absolute_path)
self.update_topic_list(dm.judged_topics())
self.update_document_list([])
#########################################################################################
# Updateable UI Elements #
#########################################################################################
def update_topic_list(self, topics):
'''
Updates the topics displayed in Topic View.
'''
self._topics = topics
self._topic_list.clear()
self._topic_list.addItems([t.id for t in topics])
self._topic_list.sortItems()
def update_document_list(self, documents):
'''
Updates the documents displayed in Document View.
'''
self._document_list.clear()
self._document_list.addItems(documents)
self._document_list.sortItems()
def update_rationale_selection(self, rationales):
'''
Regenerates data structures and display logic for rationale selection.
'''
# Compact holder for all rationale logic.
container = namedtuple('RationaleContainer', ['rationale', 'color', 'display'])
# Friendly display colors used to highlight rationales in source texts.
colors = [QtGui.QColor(102, 255, 102), # Light Green
QtGui.QColor(255, 102, 102), # Light Red
QtGui.QColor(102, 201, 255), # Electric Blue
QtGui.QColor(102, 178, 255), # Baby Blue
QtGui.QColor(178, 102, 255), # Light Purple
QtGui.QColor(255, 205, 255), # Light Magenta
QtGui.QColor(255, 102, 178), # Light Pink
QtGui.QColor(192, 192, 192)] # Light Grey
# Remove old rationale widgets.
layout = self._selection_layout
for i in reversed(range(layout.count())):
layout.itemAt(i).widget().setParent(None)
# Re-generate rationale data structures.
self._rationales = []
for i in range(len(rationales)):
r = rationales[i]
c = colors[min(i, len(colors) - 1)]
d = QCheckBox(r.label)
d.stateChanged.connect(self._rationale_selection_changed)
self._rationales.append(container(rationale = r, color = c, display = d))
self._selection_layout.addWidget(d)
def update_rationale_display(self):
'''
Recomputes rationale overlap and updates display. Expensive.
'''
print ("Updating rationale display...")
# Disable rationale selection while updating.
for this_rationale in self._rationales:
this_rationale.display.setEnabled(False)
# Locate selected rationales.
selected = []
for this_rationale in self._rationales:
if this_rationale.display.isChecked():
selected.extend([r for r in self._rationales if this_rationale.display.text() == r.rationale.label])
print (selected)
# Gather text.
text = self._rationale_display.get_text()
# Compute overlap.
just_rationales = [r.rationale for r in selected]
result = Rationale.compute_overlap(text, just_rationales)
# Update display with rationale matches.
display = self._rationale_display
display.clear()
# If more than one rationale is selected, highlight overlap.
'''
if result.overlap:
for string in result.overlap:
print ("Highlighted string: %s" % string)
display.highlight(string)
# Else, highlight single rationale
elif result.matches:
strings = [QtCore.QString(s) for s in itertools.chain(*[s for s in result.matches.values()])]
for string in strings:
print ("Highlighting string: %s" % string)
display.highlight(string)
'''
for r in just_rationales:
display.highlight(r.rationale.rationale)
# This is code exclusively for a multi-color highlight interface.
'''
for rationale, matches in result.matches.items():
# Map back to container to find correct color.
color = [c.color for c in selected if c.rationale is rationale][0]
# Highlight in display.
for match in matches:
display.highlight(match, color)
overlap_color = QtGui.QColor(255, 255, 102) # Light Yellow
for string in result.overlap:
display.highlight(string, overlap_color)
'''
# Re-enable rationale selection.
for this_rationale in self._rationales:
this_rationale.display.setEnabled(True)
def update_statistics(self, topic=None, document=None):
self.update_gold_standard_view(topic, document)
self.update_topic_view(topic)
self.update_rationale_list()
self.update_judgment_list()
self.update_confusion_matrix(topic, document)
self.update_agreement_view(topic, document)
def update_confusion_matrix(self, topic=None, document=None):
cm = self._dm.confusion_matrix(topic, document)
self._confusion_matrix.setText(cm)
def update_gold_standard_view(self, topic, document):
'''
Updates the current gold standard view.
'''
if topic and document:
value = self._dm.gold_standard(topic, document)
self._gold_standard_view.setText("Gold Standard: %s" % value)
def update_topic_view(self, topic):
'''
Updates the current topic and rationale display.
'''
(topic, narrative) = self._dm.topic_information(topic)
self._topic_view.setText("<b>Topic</b>: %s" % topic)
self._narrative_view.setText("%s" % narrative)
def update_agreement_view(self, topic, document):
'''
Computes and updates the agreement for currently selected Topic or Document.
'''
d1_agree = self._dm.agreement(1, topic, document)
self._d1_agreement_view.setText("D1 Agreement: %f" % d1_agree)
d2_agree = self._dm.agreement(2, topic, document)
self._d2_agreement_view.setText("D2 Agreement: %f" % d2_agree)
def update_rationale_list(self):
'''
Updates the worker rationale list. If one or more worker IDs is
selected, this will display only the rationales by the selected
workers. Otherwise, this will display all rationales for the
selected Topic and Document.
'''
display_text = ''
selected = [r for r in self._rationales if r.display.isChecked()]
# If none are selected, display all.
selected = selected if selected else self._rationales
for r in selected:
display_text += ('%s\n\n%s\n\n' % (r.rationale.label, r.rationale.rationale.rationale))
self._worker_rationales.setText(display_text)
def update_judgment_list(self, topic=None, document=None):
'''
Updates the worker judgment list. If one or more worker IDs is
selected, this will display only the judgments of those selected
workers. Otherwise, this will display the judgments from all workers.
'''
display_text = ''
selected = [r for r in self._rationales if r.display.isChecked()]
# If none are selected, display all.
selected = selected if selected else self._rationales
for r in selected:
display_text += ('%s: %s\n' % (r.rationale.label, r.rationale.rationale.value))
self._worker_judgments.setText(display_text)
def update_rationale_text(self, text):
'''
This updates the text in the rationale display.
'''
self._rationale_display.set_text(text)
def highlight_rationale(self, text):
self._rationale_display.highlight(text)
def load_document(self, url):
self._rationale_display.load(QUrl(url))
#########################################################################################
# Signals #
#########################################################################################
def _topic_selected(self, item):
'''
Handler function - user selects a topic in the Topic View.
Refreshes the list of documents in the Document View with
all documents for which a worker judgment has been loaded
for that topic. Computes statistics across that topic.
'''
topic_id = item.text()
# Update control selection.
self._selected_topic = topic_id
# Update statistics view.
self.update_statistics(topic=str(topic_id))
print ("Loading documents for topic %s" % topic_id)
documents = self._dm.judged_documents_by_topic(topic_id)
self.update_document_list([d.id for d in documents])
def _document_selected(self, item):
'''
Handler function - user select a document in the Document View.
Loads the text from that document into the rationale display
and computes statistics for that document.
'''
document_id = item.text()
# Update control selection.
self._selected_document = document_id
# Grab control selections.
selected_topic = self._selected_topic
selected_document = self._selected_document
print ("Loading rationales for document %s, topic %s" % (selected_document, selected_topic))
rationales = self._dm.judgments(selected_topic, selected_document)
rationales = [Rationale(str(random.randint(1,10000)), r) for r in rationales]
self.update_rationale_selection(rationales)
# Update statistics view.
self.update_statistics(str(selected_topic), str(selected_document))
# Load document.
document = next((d for d in self._dm.judged_documents() if d.id == selected_document), None)
self.load_document(document.url)
def _rationale_selection_changed(self, state):
'''
Handler function called when a user selects or deselects a rationale
check box.
'''
worker = Thread(target=self.update_rationale_display)
worker.start()
def FormatPlateMapData(keys_and_vals, categorical=False):
'''
keys_and_vals -- a list of lists of well-keys and values
eg: [['p1', 'A01', 0.2],
['p1', 'A02', 0.9], ...]
returns a 2-tuple containing:
-an array in the shape of the plate containing the given values with
NaNs filling empty slots. If multiple sites per-well are given, then
the array will be shaped (rows, cols, sites)
-an array in the shape of the plate containing the given keys with
(UnknownPlate, UnknownWell) filling empty slots
'''
from itertools import groupby
keys_and_vals = np.array(keys_and_vals)
nkeycols = len(dbconnect.well_key_columns())
shape = list(p.plate_shape)
if p.plate_type == '5600':
well_keys = keys_and_vals[:,:-1] # first column(s) are keys
data = keys_and_vals[:,-1] # last column is data
assert data.ndim == 1
if len(data) < 5600: raise Exception(
'''The measurement you chose to plot was missing for some spots.
Because CPA doesn't know the well labelling convention used by this
microarray, we can't be sure how to plot the data. If you are
plotting an object measurement, you may have some spots with 0
objects and therefore no entry in the table.''')
assert len(data) == 5600
data = np.array(list(meander(data.reshape(shape)))).reshape(shape)
sort_indices = np.array(list(meander(np.arange(np.prod(shape)).reshape(shape)))).reshape(shape)
well_keys = np.array(list(meander(well_keys.reshape(shape + [nkeycols] )))).reshape(shape + [nkeycols])
return data, well_keys, sort_indices
# compute the number of sites-per-well as the max number of rows with the same well-key
nsites = max([len(list(grp))
for k, grp in groupby(keys_and_vals,
lambda row: tuple(row[:nkeycols]))
])
if nsites > 1:
# add a sites dimension to the array shape if there's >1 site per well
shape += [nsites]
data = np.ones(shape) * np.nan
if categorical:
data = data.astype('object')
if p.plate_id:
dummy_key = ('UnknownPlate', 'UnknownWell')
else:
dummy_key = ('UnknownWell',)
well_keys = np.array([dummy_key] * np.prod(shape),
dtype=object).reshape(shape + [nkeycols])
sort_indices = np.ones(data.shape)*np.nan
dm = DataModel.getInstance()
ind = keys_and_vals.argsort(axis=0)
for i, (k, well_grp) in enumerate(groupby(keys_and_vals[ind[:,len(dummy_key)-1],:],
lambda row: tuple(row[:len(dummy_key)]))):
(row, col) = dm.get_well_position_from_name(k[-1])
well_data = np.array(list(well_grp))[:,-1]
if len(well_data) == 1:
data[row, col] = well_data[0]
sort_indices[row,col] = ind[:,len(dummy_key)-1][i]
else:
data[row, col] = well_data
sort_indices[row,col] = ind[:,len(dummy_key)-1][i*nsites + np.array(range(nsites))]
well_keys[row, col] = k
return data, well_keys, sort_indices
def main(args):
dm = DataModel()
dm.read_data(to_read_count=10, normalize_data=True)
dm.write_data('../data/data_imp.csv')
def captureevents(self):
"""
Capture BMC/IPMI System Event Log entries (SEL) and put into common DB.
:param: none
:returns: status, reply
:raises: None
Make sure IPMI is present.
Get the IPMI sel logs.
There may be more than 1 line (usually come in doubles).
Insert into the common database. That routine eliminates duplicates.
When finished, clear all SEL logs.
"""
# Only do this if ipmi is in the kernel and active. (Ignore VM's)
if not self.ipmisupported:
return 0, ''
reply = "Unable to capture IPMI/BMC System Event Logs (SEL)"
# Get the BMC/IPMI System Event Logs (SEL).
try:
# stderr=subprocess.PIPE and ignoring sel_lines.stderr -- tosses the output.
sel_lines = subprocess.check_output(['/usr/bin/ipmitool', 'sel', 'list'], stderr=subprocess.PIPE)
except subprocess.CalledProcessError as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
# If None, leave.
if sel_lines is None:
return 0, ''
# Convert output to usable format.
sel_lines = sel_lines.decode('utf-8').splitlines()
# If nothing in it, leave -- nothing new to enter into common database.
if sel_lines == []:
return 0, ''
# Decode the SEL logs, place into format for SQL table, and put there.
for a in sel_lines:
# Split line into list on space pipe space.
selarr = a.split(' | ')
# Date and time are separated by ' | ', combine and separate with space.
dt = selarr[1] + ' ' + selarr[2]
# Get 'date time' in SQL format datetime.
selarr[2] = parse(dt)
# List slicing to remove elements 0 and 1.
selarr = selarr[2:]
done = False
# Get events stored in common database.
table_name = 'ipmievents'
sqlcommand = ("SELECT * FROM {0} WHERE (datetime='{1}' and event='{2}' and details='{3}' and status='{4}')"
.format(table_name, selarr[0], selarr[1], selarr[2], selarr[3]))
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
except Exception as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
for l in ipmidata:
done = True
break
# New entry, insert into database.
if not done:
query = "INSERT INTO ipmievents (datetime, event, details, status) " \
"VALUES ( '{0}', '{1}', '{2}', '{3}');" \
.format(selarr[0], selarr[1], selarr[2], selarr[3])
try:
DataModel().ExecuteRawQueryStatement(query)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
except Exception as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
# Clear all SEL logs. Note: minor race condition from read to clear is possible.
try:
output = subprocess.check_output(['/usr/bin/ipmitool', 'sel', 'clear'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
print("error in ipmitool sel clear") # DEBUG
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
# Everything is good at this point.
return 0, ''
def __init__(self, properties=None, parent=None, id=ID_IMAGE_GALLERY, **kwargs):
if properties is not None:
global p
p = properties
global db
db = dbconnect.DBConnect.getInstance()
wx.Frame.__init__(self, parent, id=id, title='CPA/ImageGallery - %s' % \
(os.path.basename(p._filename)), size=(800, 600), **kwargs)
if parent is None and not sys.platform.startswith('win'):
self.tbicon = wx.TaskBarIcon()
self.tbicon.SetIcon(icons.get_cpa_icon(), 'CPA/ImageGallery')
else:
self.SetIcon(icons.get_cpa_icon())
self.SetName('ImageGallery')
db.register_gui_parent(self)
global dm
dm = DataModel.getInstance()
if not p.is_initialized():
logging.critical('ImageGallery requires a properties file. Exiting.')
raise Exception('ImageGallery requires a properties file. Exiting.')
self.pmb = None
self.worker = None
self.trainingSet = None
self.classBins = []
self.binsCreated = 0
self.chMap = p.image_channel_colors[:]
self.toggleChMap = p.image_channel_colors[
:] # used to store previous color mappings when toggling colors on/off with ctrl+1,2,3...
self.brightness = 1.0
self.scale = 1.0
self.contrast = 'Linear'
self.defaultTSFileName = None
self.defaultModelFileName = None
self.lastScoringFilter = None
self.menuBar = wx.MenuBar()
self.SetMenuBar(self.menuBar)
self.CreateMenus()
self.CreateStatusBar()
#### Create GUI elements
# Top level - three split windows
self.splitter = wx.SplitterWindow(self, style=wx.NO_BORDER | wx.SP_3DSASH)
self.fetch_and_rules_panel = wx.Panel(self.splitter)
self.bins_splitter = wx.SplitterWindow(self.splitter, style=wx.NO_BORDER | wx.SP_3DSASH)
# fetch & rules
self.fetch_panel = wx.Panel(self.fetch_and_rules_panel)
self.find_rules_panel = wx.Panel(self.fetch_and_rules_panel)
# sorting bins
self.gallery_panel = wx.Panel(self.bins_splitter)
o_label = p.object_name[0] if p.classification_type == 'image' else '' + ' image gallery'
self.gallery_box = wx.StaticBox(self.gallery_panel, label=o_label)
self.gallery_sizer = wx.StaticBoxSizer(self.gallery_box, wx.VERTICAL)
self.galleryBin = sortbin.SortBin(parent=self.gallery_panel,
classifier=self,
label='image gallery',
parentSizer=self.gallery_sizer)
self.gallery_sizer.Add(self.galleryBin, proportion=1, flag=wx.EXPAND)
self.gallery_panel.SetSizer(self.gallery_sizer)
self.objects_bin_panel = wx.Panel(self.bins_splitter)
# fetch objects interface
self.startId = wx.TextCtrl(self.fetch_panel, id=-1, value='1', size=(60, -1), style=wx.TE_PROCESS_ENTER)
self.endId = wx.TextCtrl(self.fetch_panel, id=-1, value='100', size=(60, -1), style=wx.TE_PROCESS_ENTER)
self.fetchChoice = wx.Choice(self.fetch_panel, id=-1, choices=['range','all','individual'])
self.fetchChoice.SetSelection(0)
self.filterChoice = wx.Choice(self.fetch_panel, id=-1,
choices=['experiment'] + p._filters_ordered + p._groups_ordered + [
CREATE_NEW_FILTER])
self.fetchFromGroupSizer = wx.BoxSizer(wx.HORIZONTAL)
self.fetchBtn = wx.Button(self.fetch_panel, -1, 'Fetch!')
#### Create Sizers
self.fetchSizer = wx.BoxSizer(wx.HORIZONTAL)
self.find_rules_sizer = wx.BoxSizer(wx.HORIZONTAL)
self.fetch_and_rules_sizer = wx.BoxSizer(wx.VERTICAL)
self.classified_bins_sizer = wx.BoxSizer(wx.HORIZONTAL)
#### Add elements to sizers and splitters
# fetch panel
self.fetchSizer.AddStretchSpacer()
self.fetchSizer.Add(wx.StaticText(self.fetch_panel, -1, 'Fetch '), flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchSizer.Add(self.fetchChoice, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchTxt = wx.StaticText(self.fetch_panel, -1, label='of image IDs:')
self.fetchSizer.Add(self.fetchTxt, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchSizer.Add(self.startId, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchTxt2 = wx.StaticText(self.fetch_panel, -1, label='to')
self.fetchSizer.Add(self.fetchTxt2, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchSizer.Add(self.endId, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
#self.fetchSizer.Add(self.obClassChoice, flag=wx.ALIGN_CENTER_VERTICAL)
#self.fetchSizer.AddSpacer((5, 20))
self.fetchTxt3 = wx.StaticText(self.fetch_panel, -1, label='images')
self.fetchSizer.Add(self.fetchTxt3, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchSizer.Add(wx.StaticText(self.fetch_panel, -1, 'from'), flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchSizer.Add(self.filterChoice, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((10, 20))
self.fetchSizer.Add(self.fetchFromGroupSizer, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddSpacer((5, 20))
self.fetchSizer.Add(self.fetchBtn, flag=wx.ALIGN_CENTER_VERTICAL)
self.fetchSizer.AddStretchSpacer()
self.fetch_panel.SetSizerAndFit(self.fetchSizer)
# fetch and rules panel
self.fetch_and_rules_sizer.Add((5, 5))
self.fetch_and_rules_sizer.Add(self.fetch_panel, flag=wx.EXPAND)
self.fetch_and_rules_sizer.Add((5, 5))
self.fetch_and_rules_panel.SetSizerAndFit(self.fetch_and_rules_sizer)
# classified bins panel
self.objects_bin_panel.SetSizer(self.classified_bins_sizer)
# splitter windows
self.splitter.SplitHorizontally(self.fetch_and_rules_panel, self.bins_splitter,
self.fetch_and_rules_panel.GetMinSize()[1])
self.bins_splitter.SplitHorizontally(self.gallery_panel, self.objects_bin_panel)
self.splitter.SetSashGravity(0.0)
self.bins_splitter.SetSashGravity(0.5)
self.splitter.SetMinimumPaneSize(max(50, self.fetch_and_rules_panel.GetMinHeight()))
self.bins_splitter.SetMinimumPaneSize(50)
self.SetMinSize((self.fetch_and_rules_panel.GetMinWidth(), 4 * 50 + self.fetch_and_rules_panel.GetMinHeight()))
# Set initial state
self.filterChoice.SetSelection(0)
# JEN - Start Add
# self.openDimensReduxBtn.Disable()
# JEN - End Add
self.fetchSizer.Hide(self.fetchFromGroupSizer)
#####################
#### GUI Section ####
#####################
# add the default classes
#for class in range(1, num_classes+1):
self.AddSortClass('objects of selected image')
#self.AddSortClass('negative')
self.Layout()
self.Center()
self.MapChannels(p.image_channel_colors[:])
self.BindMouseOverHelpText()
#self.Bind(wx.EVT_BUTTON, self.OnInspect, self.inspectBtn)
# JEN - Start Add
# self.Bind(wx.EVT_BUTTON, self.OpenDimensRedux, self.openDimensReduxBtn)
# JEN - End Add
self.Bind(wx.EVT_BUTTON, self.OnFetch, self.fetchBtn)
self.startId.Bind(wx.EVT_TEXT, self.ValidateIntegerField)
self.startId.Bind(wx.EVT_TEXT_ENTER, self.OnFetch)
self.Bind(wx.EVT_CLOSE, self.OnClose)
self.Bind(wx.EVT_CHAR, self.OnKey) # Doesn't work for windows
tilecollection.EVT_TILE_UPDATED(self, self.OnTileUpdated)
self.Bind(sortbin.EVT_QUANTITY_CHANGED, self.QuantityChanged)
self.Bind(wx.EVT_CHOICE, self.OnSelectFetchChoice, self.fetchChoice)
self.Bind(wx.EVT_CHOICE, self.OnSelectFilter, self.filterChoice)
from sys import stderr
from tempfile import gettempdir
from time import ctime, time
#from wx.lib.embeddedimage import PyEmbeddedImage
import dbconnect
import imagetools
import csv
import logging
import numpy as np
import os
import sys
import weakref
import wx
import wx.grid
dm = DataModel.getInstance()
db = dbconnect.DBConnect.getInstance()
p = Properties.getInstance()
ID_LOAD_CSV = wx.NewId()
ID_SAVE_CSV = wx.NewId()
ID_EXIT = wx.NewId()
DO_NOT_LINK_TO_IMAGES = 'Do not link to images'
ROW_LABEL_SIZE = 30
# Icon to be used for row headers (difficult to implement)
#img_icon = PyEmbeddedImage('iVBORw0KGgoAAAANSUhEUgAAABUAAAASCAYAAAC0EpUuAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOR2lDQ1BQaG90b3Nob3AgSUNDIHByb2ZpbGUAAHjarZdXUNRbl8X3v5tuUpObnJqcsySRnARRckZSkyW0bYMgIiogWTKiIBlEiQoogiJJRC6CCCqgggiCERFUkNTfg3furZqqmXmZ9XDqt1edqrP2edm1AVgO+pBIoSgACAunkO3MjQgurm4E2mmgB25gBQREfIgnSIY2NlbwP+rXC0AAACYUfEikUAuqw7tL/mLqKZ9WtIcbq2fhfxcz2cXVDQCRBwB84B82AAC87x92AAD8SQqJAoAEAQCeGOTjB4CcBgB5soOdMQBSBwDMgX+4AwCYff/wEAAwRxEDKQDIFACWPdwvOByA9gsAVs/P/wQRgFkeAPz8ThDDAJizASA0LCzCD4B5BwCkiSQyBYBFHADEXVzdCH8iewOA+gAArey/XkQOQCsRgGftX0/KBIBrHaCV/l9v3Q4QAEC4xk4EqKkCAACCMwLAzFKp65IAtLkAuzlU6nYllbpbBYB+DdATSowkR/39XwjyBOD/qv/0/LfQCAAKAGQgGyEgnagotA2NKcYHm0/7ht6QoROnzdTL4sz6gT2KY48zhmuJx5K3mG9RgFNQQ8hNOIZQJNIhOi1GlRCVNJMKkc6W6ZJ9L8+qYKxIVqpRfqXKomaxL0l9QBO0TLUT9j/Q+azLprdf380g1rDc6KHxkimDmaK57cHjFrmWtw9NWW0fEbDeb+NqG2dXYt/lMOO45SzoYuIa4VbpPnkU47nPy887x6fXd81P1N8xICtwIGg3ROdYfGh3OCbCmnT1+MoJY0pB5OpJg2jfmNBT5NiY0wlxF86kx6edzTiXfD4uIToxKMnxglGyTApTypfUR2kV6ZEZlpl8mcsXW7POZjvlKOTS5L7Ku5WfUuBZqHqJ9tKbotbLqVd8irVKWEqWr/aWlpRFlztW7KvEV/6sel7dUZNR63RN7NpaXc/1zBse9XL1mw2PGouagpr3tzC0zNxsuBXXeqSN0LbW/vB2/h1ih9pd9N2nndldh+/R3Ru4f77boJv6oLvndK9uH/T19ScNWD3kePhqsObR8SHdx4yPXw03/nV2xP6JxJMfo91jKU9tx3nH3z1rmjg5qf+c4fnzF+UvQ6e0p2mnn89Uv6K8NnnD+ebjbOdc1lvivMY7pnefFvoWC977L6ksbS8//pD/kfhJ4zPT5w9fxr8Orgx+m1qlrhmtX/7J8Kt602PLbMd/L5dKBQBDWESOo7hRE+g7NC2YR9gNOl36AkYsLpLpB0scG7Cf59jjDOca45Hljea7w78kiBWSFj5I8BM5J1oh9lB8URIrJS1tJhMhmyfXIb+gyKykqeynclG1W21VXVzDXjNN6572Zx26A4K66noH9V0Mwg0TjAqNb5jcNx0zmzdfs8Ba8hySsdp/2OKIs3WIzUnbRLtc+wqHm46DTu9cUK5ibsbuPh7njpZ5dnvNeVN9RYiWfhH+lwL6A38Ei4Y4HssMHQxHRxiTzhwfIO9QOCMFooRPikVLxUifko6VOi0ZJ3qGL57jLP3ZnXMr52cTniR2JJVdSEsOS7FP1UjjTttIn8ioz0y66J6llo3LXsrpyb2SR863KVAoxBV+uTRS1HQ560pksXuJ8VW5Uq4yVNlK+UzFYGVlVVi1bg1rzUJt+7WMOu/r6jeYbryv72640khuOtQs0Uxtmb7Zciutldim1Y5rf3u77U5Kh+ddtU5s51RXw734+/bdEt0bD4Z7rvaG9xn04/uXB+4+zBz0eaQ+xDA097h9OPMvvxH9J+KjHGPosbWnb8aHnrVNlEwmPA95YfdSe0p4mnb628zUq77XzW+KZ9Pn4t5GzPu/c1+wWzz03mzJZNnwg/5H40+Wn+2+eHwNWTn1LWu17nvP2sz6xk++Xwc2vDYTftdvPdve2BXa06caUqkAwA+e0I9oIE0oPdQi+goNEaOHlaUVoOOl52MQZRTHyTPJMYuzsLNss86wNbFHcWhx/Ma3c5K5lLm+cTfzhPMq8q7ytfJHCegIIoIjQoXCPgQlwrbIiGiZGFncVEJIYl2yVypRWk96S6ZNNlxOTu6DfK1CgKKk4rJSnXKIiorKT9VOtbP7zNQZ1Sc1ijV9tKS1VrU79sfrmB1gPTCjW60Xrq9pgBiMGF4yIhorm4DJqOlVswhzk4P4gx8s7lvmHzpmZXKY//CPI4+tC2w8bMVsv9jdtj/vYOso5Ljq1Odc6BLhauYm4kZ1n/PoOVrhmeAV7G3lo+7LT0SIH/yG/esCLgT6BBkECwdTQ94e6w2tCLsQfizCmqR5XIiMJa+ceEnpi2yIKjqZGH08xuuURazKaeT0QFzCGdN4dHzP2fhzWufWzjckBCaKJr5JKrpgn8yS/CQlI9UyjTFtND07wy6TO/P1xYost2yO7L9yEnMN8yCvNz+xwKKQvXDuUlNR/GXXK2rFrMWrJZNXO0uryrLK4yoiKn2qjlSr1rDWfKp9eK2i7sx1rxv69YQGdMPHxtGm1ubSluSbpFvuraZtyu38t7G3V++87nh4t6mztKvn3mI38oC3R63Xvi+mv27g3aDso/ihL8OkEdonD8aqx9smNl7ETLu9Ln2b9/7I5+qfllQqwJ/ZBwCA1QC4jAJwGgJwpAHIaQGQygHg9AawYQJw0AYUgy4gW/OASAb8Mz84QB5MwQuiIQfq4SHMwW8EjygjVkgIkozUIoPIMooBpYhyQJ1GXUNNoNFodXQIuga9SCNOE0jTQLOO0cEkYcawAtgQbCctjvYobQsdhs6droWejt6HvpMBz3Cc4QmjLGMy4xLOHFfLRM8UyvSc2YS5nUWSpYQVz5rORsMWz7bLfop9iyOWg4pP4mTgzOHi56rhVuXu47HneccbxcfIV8m/n39CIFSQRrBESEXokbC38Bbhksg+kUlRihhe7La4m/iuxFVJI8llqXRpRekxmUhZXtkuOU95kK9UMFFYUkxXklcaVT6lYqIqqPpb7eW+VvUijVjNQC1bbeP9ujpqBzR09fUs9b0NYg0rjUZMUKbaZmfMhyzYLW0PZVtNHuG19rVptN2zt3aocNxwtnSpct11d/Ro9MR6eXnf8WUhhvr1B2gG9gYbhQyF2ofNRASTNslJFM7ImpMHosdPhZxGx5XH65ydP5+aqJr0Nvlmalb6iUy7LLUcfO52/kzh/aLSK9ElzqXK5XQVc1VtNReuOV2XqadtWGtaaJm69axt4HZfx4POu/c6utt7bvU1DzQO3hi6PlwxUjVa8rT02dXJtBfJU7kzDa+fzX6b511Qe2+0bP5R+RP1S9eK+7f574fWatdXf8r8stzw3Qz/HbTlvK2xw7ozu1u9d4lKBQAciMEBcIBwSIYK6IIX8B3BITKIGeKHnEcqkX7kPYoepYhyRMWj6lHTaHq0LpqCbkGv0qjQUGg6MSjMIUwB5i1WCXsGO0YrRhtJO0wnShdDN04vR59Mv8BgxFDBCIw+jH04KVwabpXJhamfWYu5hUWapZpVjLWCTYStml2avYlDg6MHb4Wf4Qzi/M2VwS3CfZfHnuczbyqfNN9j/mMCzAI3Be0EvwtlC6sITxJiRIRFHomSxPjEHoqTJYQkhiSjpMSlnknHycjKzMgmy6nJvZXPUNBWeK9YqGSotKrcqHJO1VNNb5+wOqL+UWNcs1urXrt8f5FO9oEc3Ty9Yv0ag3bDUaNFE4ypuJmV+cmDtRYLh4SsfA9XHVmykbM9bnffAXG0dMpxnnEVdQt3b/PY8TTxyvCe8OUl+vs1BLAHXg7Gh2SHsoVlRDCTLpJxJzIi6aOyovExxbFip9vOGMRPnCMlsCbeuuCYvJFanm6dsXOxMds7ly9vvCDj0pHLuCsjJXmlTuUCFYtVDTWUa/uvY270NRCbUM3Xbtq1otqabwd3SNyd7yq77/dAvOdD342B8EG5R98e1/1FfCI4+uxp+jODiY3ntS+dp3Ezd197z2LmGuet331dTFsiLN/6aPlp7gvp6/a3lO9cazU/lH92bJhvvtjy297dvUilAoAgmEAI5EAHzCF0iCriiaQj95AVlDjKE1WKeoeWR8ejX9Po0dRi2DHnMOvYEOwCrRftLJ033Tx9IP0KQwwjmjELR8A1MxkzzTPnspiy7LF2sFHY1dh/cNzBn+E05mLimuKu4znFa8UnwUflfyXQKVgmdF44iGAroieqICYoziaBkdiU/C71UXpO5o3sS7lJ+XGFMcVnShPKi6o0alL7Dqof00jXbNYa1V7TYT6gpHtYL1g/yaDSsM/ojfGeKZ+Zrrn3wSSLDsuvVjKHfY5csX5uy2pna5/h8NiJ3tnEJcl10J3Ow+JohudTb04fN99y4oq/UcB4EDmE5di9sIAIYdI0OZ9iF4U/OR1TEusRJ3pm9mzFeb9E8aSl5EepLenFmbFZLjnKedj80cKCIqcrtMWdVwPLmMtvVdpVfatJuSZU13bDtYGzcba57ialVbcdbj/uyOq0ucd+f+pBfq9zP8/AzGDpkP+w9F8/nnSNpY97TohO5r+gf5k6zTFT+lr8TdUc/9v0+e8LVotX3y8s83ww+Oj7KfZzwpekr+SVo980V1lWX34vW3NeZ17v+uHy49fP9F/cv6o2pDdKNlk2Eza//7b73bbFtRW9NbzNse2xXbO9sqO5E7VTu/N8F7O7b9dvN2+3f3d9T3TPeu/UXs3e071tqhTVjhpHraNOUqkAf/YlAABgMI4IjSATrIxN4P9XYaGR//UGGgBw/uGO9gDADgD7Ashmdn+zWTDFwgEA8ADgAsYQAaEQAWQggBUYgwk8BiKQwQfC/3H+nAQg/nM3+M+eBwCAZQUoOQcA0Hufeva/Z6L4R1MAAIwjSDHk4MAgCsGQRAr1lydYhBMV5Qmqysra8B9BJBMdj2+jxwAAACBjSFJNAABtmAAAc44AAONlAACDMAAAfIYAANfxAAAzGQAAGuA00fGBAAACJUlEQVR42qyUPUsjURSGnxOjSKYQSZMmTVBQQZgldVAwTa6wv2C74I+wC9jKdnamyy/YZpoIgRRbGjaIhWuRbiur+cqM49lmZnDyARvYA5fLXN773vPxviOqyroQER4eHtYDFuLi4kIAyv8CHo/Hhe8kSZjP5wRBgO/7eJ7H1dUVQA/olUWETSKKIsIwxPd9fN/HdV1c1y1gVma6WHKr1Vr7yM3NzUrSb8A5MAIGm5b8/PyMZVlLpOeq2k3bMNi05CAIeH9/XyIdiQjX19fdyWTSBbi9vS2AbNvm8vKS4XDIbDZjMpkQxzHD4ZAgCHBdN7vzFfgNcK+q2ul0dF2cnJzo2dmZNhqNwnmn01GgcAbclxZLbTabNJtNAE5PTzk+Pubp6YnZbMbr6yvGGHZ2dnK8qiIiGGMQkUdgtDR9VSVJkryXWb/m83mO+fj4KBgkM1AmzyXSOI5zoiiKiON4iTR7FODg4ABjTJbQFxE5L6+a9OfsPpPu7u4ShmEB//Ly8jnrR2BUSqfft22bw8ND9vb2sCyLWq3G9vY2pVKJSqWSyyrtXYFIRPoi0ge+A4Nyqs1Bu93uTqdTXNdFVdnf38fzPJIkQUTyPeufMQbbtnEc50fq+dU2fXt7y4WdCT0MQ6IoyjHGGBzH+QP8dBzn1+KQl/x+dHSk9Xpdq9WqVioV3draUqCw0v9Cb5XzVBVVLWZ6d3fH/wjJxJtGb4O7vZUlA38HAO/oekRA0FPwAAAAAElFTkSuQmCC')
class HugeTable(wx.grid.PyGridTableBase):
'''
def score_objects(properties, ts, gt, nRules, filter_name=None, group='Image',
show_results=False, results_table=None, overwrite=False):
'''
Trains a Classifier on a training set and scores the experiment
returns the table of scores as a numpy array.
properties -- Properties instance
ts -- TrainingSet instance
gt -- Ground Truth instance
nRules -- number of rules to use
filter_name -- name of a filter to use from the properties file
group -- name of a group to use from the properties file
show_results -- whether or not to show the results in TableViewer
results_table -- table name to save results to or None.
'''
p = properties
#db = DBConnect.getInstance() ## Removed writing to db. Results_table should be 'None' anyway
dm = DataModel.getInstance()
#if group == None:
#group = 'Image'
if results_table:
if db.table_exists(results_table) and not overwrite:
print 'Table "%s" already exists. Delete this table before running scoreall.'%(results_table)
return None
print ''
print 'properties: ', properties
print 'initial training set: ', ts
print 'ground truth training set: ', gt
print '# rules: ', nRules
print 'filter: ', filter_name
print 'grouping by: ', group
print 'show results: ', show_results
print 'results table: ', results_table
print 'overwrite: ', overwrite
print ''
nClasses = len(ts.labels)
nKeyCols = len(image_key_columns())
assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s'%(nRules,)
assert filter_name in p._filters.keys()+[None], 'Filter %s not found in properties file. Valid filters are: %s'%(filter_name, ','.join(p._filters.keys()),)
assert group in p._groups.keys()+['Image', 'None'], 'Group %s not found in properties file. Valid groups are: %s'%(group, ','.join(p._groups.keys()),)
output = StringIO()
logging.info('Training classifier with %s rules...'%nRules)
t0 = time()
weaklearners = fastgentleboostingmulticlass.train(ts.colnames,
nRules, ts.label_matrix,
ts.values, output)
logging.info('Training done in %f seconds'%(time()-t0))
t0 = time()
#def update(frac):
#logging.info('%d%% '%(frac*100.,))
## Score Ground Truth using established classifier
gt_predicted_scores = per_cell_scores(weaklearners, gt.values, gt.colnames)
#plt.hist(gt_predicted_scores)
#plt.show()
gt_predicted_signs = np.sign(gt_predicted_scores)
## Compare Ground Truth score signs with the actual ground truth values
numclasses = ts.labels.size
gt_actual_signs = gt.label_matrix[:,0]
cm_unrotated = metrics.confusion_matrix(gt_actual_signs,gt_predicted_signs)
## sklearn.metrics.confusion_matrix -- 2D confusion matrix is inverted from convention.
## https://github.com/scikit-learn/scikit-learn/issues/1664
cm = np.rot90(np.rot90(cm_unrotated))
fpr, sens, thresholds = metrics.roc_curve(gt_actual_signs,gt_predicted_signs)
spec = 1-fpr
s = np.sum(cm,axis=1)
percent = [100*cm[i,i]/float(s[i]) for i in range(len(s))]
avg = np.mean(percent)
avgTotal = 100 * np.trace(cm) / float(np.sum(cm))
print 'accuracy = %f' % avgTotal
print 'Confusion Matrix = ... '
print cm
my_sens = cm[0,0] / float(cm[0,0] + cm[0,1]) #TP/(TP+FN)
my_spec = cm[1,1] / float(cm[1,1] + cm[1,0]) #TN/(TN+FP)
print 'My_Sensitivity = %f' % my_sens
print 'My_Specificity = %f' % my_spec
print 'Sensitivity = ...'
print sens
print 'Specificity = ...'
print spec
print 'Done calculating'
############
## Confusion Matrix code from here: http://stackoverflow.com/questions/5821125/how-to-plot-confusion-matrix-with-string-axis-rather-than-integer-in-python
conf_arr = cm
norm_conf = []
## This normalizes each *row* to the color map, but I chose to normalize the whole confusion matrix to the same scale
##for i in conf_arr:
##a = 0
##tmp_arr = []
##a = sum(i, 0)
##for j in i:
##tmp_arr.append(float(j)/float(a))
##norm_conf.append(tmp_arr)
norm_conf = conf_arr / float(np.max(conf_arr))
if DISPLAY_CONFUSION_MATRIX:
fig = plt.figure()
plt.clf()
ax = fig.add_subplot(111)
ax.set_aspect(1)
res = ax.imshow(np.array(norm_conf), cmap=plt.cm.jet,
interpolation='nearest')
width = len(conf_arr)
height = len(conf_arr[0])
for x in xrange(width):
for y in xrange(height):
ax.annotate(str(conf_arr[x][y]), xy=(y, x),
horizontalalignment='center',
verticalalignment='center')
cb = fig.colorbar(res)
#cb.set_cmap = [0,1]
if width == 2 and height == 2:
plt.xticks([0,1],['FP','TN'])
plt.yticks([0,1],['TP','FP'])
else:
alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
plt.xticks(range(width), alphabet[:width])
plt.yticks(range(height), alphabet[:height])
plt.show()
print 'Done'
class ParseDlg(QDialog):
def __init__(self, filename, parent=None):
super(ParseDlg, self).__init__(parent)
self.filename = filename
self.rawdata = None
f = open(self.filename)
data = f.read()
tmp = None
for delim in ['\r\n','\r','\n']:
tmp = data.split(delim)
l = len(tmp)
if (l>1):
if not tmp[l-1]:
del tmp[l-1]
self.rawdata = tmp
break
f.close()
self.parent = parent
self.header = None
self.setModal(True)
buttonBox = QDialogButtonBox()
cancelButton = buttonBox.addButton(buttonBox.Cancel)
QObject.connect(cancelButton, SIGNAL('clicked()'), self.close)
self.okButton = buttonBox.addButton(buttonBox.Ok)
QObject.connect(self.okButton, SIGNAL('clicked()'), self.doParse)
mainLayout = QGridLayout()
importBox = QGroupBox('Import')
#importBox.setSizePolicy(QSizePolicy.Maximum,QSizePolicy.Maximum)
importBoxLayout = QGridLayout()
importBoxLayout.addWidget(QLabel('Character set'),0,0)
self.charsetComboBox = QComboBox()
self.charsetComboBox.addItems(['utf-8'])
importBoxLayout.addWidget(self.charsetComboBox,0,1)
importBox.setLayout(importBoxLayout)
separatorBox = QGroupBox('Separator Options')
#separatorBox.setSizePolicy(QSizePolicy.Maximum,QSizePolicy.Maximum)
separatorBoxLayout = QGridLayout()
separatorBoxLayout.setHorizontalSpacing(60)
self.delimTab = QRadioButton('Tab')
self.delimOther = QRadioButton('Other')
self.otherLineEdit = QLineEdit(',')
self.otherLineEdit.setMaxLength(1)
separatorBoxLayout.addWidget(self.delimTab,0,0)
separatorBoxLayout.addWidget(self.delimOther,0,2)
separatorBoxLayout.addWidget(self.otherLineEdit,0,3)
separatorBox.setLayout(separatorBoxLayout)
otherBox = QGroupBox('Other Options')
#otherBox.setSizePolicy(QSizePolicy.Maximum,QSizePolicy.Maximum)
otherBoxLayout = QGridLayout()
otherBoxLayout.setHorizontalSpacing(60)
self.quotedFieldCheckBox = QCheckBox('Quoted field as text')
self.textLabel = QLabel('Quote character')
self.textLabel.setAlignment(Qt.AlignRight|Qt.AlignVCenter)
self.textLineEdit = QLineEdit('"')
self.textLineEdit.setMaxLength(1)
self.headerCheckBox = QCheckBox('First row as header')
self.skipCommentsCheckBox = QCheckBox('Skip comment lines')
self.commentLabel = QLabel('Comment prefix')
self.commentLabel.setAlignment(Qt.AlignRight|Qt.AlignVCenter)
self.commentLineEdit = QLineEdit('#')
self.stripCheckBox = QCheckBox('Strip lines')
self.regexLabel = QLabel('Regex')
self.regexLabel.setAlignment(Qt.AlignRight|Qt.AlignVCenter)
self.stripLineEdit = QLineEdit('^\(|\)$')
otherBoxLayout.addWidget(self.quotedFieldCheckBox,0,0)
otherBoxLayout.addWidget(self.textLabel,0,1)
otherBoxLayout.addWidget(self.textLineEdit,0,2)
otherBoxLayout.addWidget(self.headerCheckBox,1,0)
otherBoxLayout.addWidget(self.skipCommentsCheckBox,2,0)
otherBoxLayout.addWidget(self.commentLabel,2,1)
otherBoxLayout.addWidget(self.commentLineEdit,2,2)
otherBoxLayout.addWidget(self.stripCheckBox,3,0)
otherBoxLayout.addWidget(self.regexLabel,3,1)
otherBoxLayout.addWidget(self.stripLineEdit,3,2)
otherBox.setLayout(otherBoxLayout)
filterBox = QGroupBox('Sample Filter')
filterBoxLayout = QGridLayout()
filterBoxLayout.setHorizontalSpacing(60)
filterBoxLayout.addWidget(QLabel('Column'),0,0)
self.columnFilterComboBox = QComboBox()
filterBoxLayout.addWidget(self.columnFilterComboBox,1,0)
filterBoxLayout.addWidget(QLabel('Sample Indicator'),2,0)
self.sampleIndicatorComboBox = QComboBox()
self.sampleIndicatorComboBox.setEnabled(False)
filterBoxLayout.addWidget(self.sampleIndicatorComboBox,3,0)
QObject.connect(self.columnFilterComboBox, SIGNAL('currentIndexChanged(int)'), self.filterColChanged)
QObject.connect(self.sampleIndicatorComboBox, SIGNAL('currentIndexChanged(int)'), self.sampleIndicatorChanged)
filterBox.setLayout(filterBoxLayout)
columnBox = QGroupBox('Column Options')
columnBoxLayout = QGridLayout()
columnBoxLayout.setHorizontalSpacing(60)
columnBoxLayout.addWidget(QLabel('Status'),0,0)
self.statusCombo = QComboBox()
columnBoxLayout.addWidget(self.statusCombo,0,1,1,2)
self.comparisonCombo = QComboBox()
self.comparisonCombo.addItem('==',operator.eq)
self.comparisonCombo.addItem('<',operator.lt)
self.comparisonCombo.addItem('<=',operator.le)
self.comparisonCombo.addItem('>',operator.gt)
self.comparisonCombo.addItem('>=',operator.ge)
columnBoxLayout.addWidget(self.comparisonCombo,0,3)
self.comparisonValue = QLineEdit('1')
columnBoxLayout.addWidget(self.comparisonValue,0,4,1,2)
columnBoxLayout.addWidget(QLabel('Gaze X'),1,0)
self.gazexCombo = QComboBox()
columnBoxLayout.addWidget(self.gazexCombo,1,1,1,2)
columnBoxLayout.addWidget(QLabel('Gaze Y'),2,0)
self.gazeyCombo = QComboBox()
columnBoxLayout.addWidget(self.gazeyCombo,2,1,1,2)
columnBoxLayout.addWidget(QLabel('Timestamp'),1,3)
self.timestampCombo = QComboBox()
columnBoxLayout.addWidget(self.timestampCombo,1,4,1,2)
columnBoxLayout.addWidget(QLabel('Trial'),2,3)
self.trialCombo = QComboBox()
columnBoxLayout.addWidget(self.trialCombo,2,4,1,2)
columnBox.setLayout(columnBoxLayout)
previewBox = QGroupBox('Preview')
previewBoxLayout = QGridLayout()
self.datatableModel = None
self.datatable = QTableView()
previewBoxLayout.addWidget(self.datatable)
previewBox.setLayout(previewBoxLayout)
mainLayout.addWidget(importBox,0,0)
mainLayout.addWidget(separatorBox,0,1)
mainLayout.addWidget(otherBox,1,1)
mainLayout.addWidget(filterBox,1,0)
mainLayout.addWidget(columnBox,3,0,1,2)
mainLayout.addWidget(previewBox,4,0,1,2)
mainLayout.addWidget(buttonBox,5,0,1,2)
self.delimTab.setChecked(True)
QObject.connect(self.headerCheckBox, SIGNAL('stateChanged(int)'), self.refreshDataTable)
QObject.connect(self.delimTab, SIGNAL('toggled(bool)'), self.updatePreview)
QObject.connect(self.delimOther, SIGNAL('toggled(bool)'), self.updatePreview)
QObject.connect(self.otherLineEdit, SIGNAL('textChanged(const QString&)'), self.otherDelimChanged)
QObject.connect(self.textLineEdit, SIGNAL('textChanged(const QString&)'), self.textChanged)
QObject.connect(self.quotedFieldCheckBox, SIGNAL('stateChanged(int)'), self.updatePreview)
QObject.connect(self.skipCommentsCheckBox, SIGNAL('stateChanged(int)'), self.updatePreview)
QObject.connect(self.stripCheckBox, SIGNAL('stateChanged(int)'), self.updatePreview)
self.skipCommentsCheckBox.setCheckState(Qt.CheckState.Checked)
self.setLayout(mainLayout)
self.setWindowTitle("Data Import - [%s]" % (self.filename))
self.show()
#self.setFixedSize(self.width(),self.height())
self.updatePreview()
def sampleIndicatorChanged(self, index):
self.updatePreview()
def filterColChanged(self, index):
if index>0:
index -= 1
self.sampleIndicatorComboBox.setEnabled(True)
items = []
data = self.parseData(tmp=True)
if self.headerCheckBox.isChecked():
data = data[1:]
for x in data:
try:
items.append(x[index])
except IndexError:
pass
items = list(frozenset(items))
items.insert(0,'')
self.sampleIndicatorComboBox.clear()
self.sampleIndicatorComboBox.addItems(items)
else:
self.sampleIndicatorComboBox.clear()
self.sampleIndicatorComboBox.setEnabled(False)
def textChanged(self):
if len(self.textLineEdit.text()) == 1:
self.updatePreview()
def otherDelimChanged(self):
if len(self.otherLineEdit.text()) == 1:
self.updatePreview()
def parseData(self, preview=False, progress_cb=None, tmp=False):
reg0 = None
if self.skipCommentsCheckBox.isChecked() and self.commentLineEdit.text():
reg0 = re.compile('^'+self.commentLineEdit.text())
reg1 = None
if self.stripCheckBox.isChecked() and self.stripLineEdit.text():
reg1 = re.compile(self.stripLineEdit.text())
lines = []
l = len(self.rawdata)
for i, line in enumerate(self.rawdata):
if progress_cb:
progress_cb(i+1,l)
line = line.encode(self.charsetComboBox.itemText(self.charsetComboBox.currentIndex()))
if reg0 and reg0.match(line):
continue
if reg1:
line = reg1.sub('', line)
lines.append(line)
if preview and len(lines) == 100:
break
quotechar = None
if self.quotedFieldCheckBox.isChecked() and self.textLineEdit.text():
quotechar = str(self.textLineEdit.text())
delim = None
if self.delimTab.isChecked():
delim = '\t'
elif self.delimOther.isChecked() and self.otherLineEdit.text():
delim = str(self.otherLineEdit.text())
tmpData = None
if not tmp and self.columnFilterComboBox.currentIndex() > 0 and self.sampleIndicatorComboBox.currentIndex() > 0:
col = self.columnFilterComboBox.itemData(self.columnFilterComboBox.currentIndex())
val = self.sampleIndicatorComboBox.itemText(self.sampleIndicatorComboBox.currentIndex())
if self.headerCheckBox.isChecked():
tmpData = [line for i,line in enumerate(csv.reader(lines, delimiter=delim, quotechar=quotechar)) if i==0 or line[col] == val]
else:
tmpData = [line for line in csv.reader(lines, delimiter=delim, quotechar=quotechar) if line[col] == val]
else:
tmpData = [line for line in csv.reader(lines, delimiter=delim, quotechar=quotechar)]
if not tmp:
self.data = tmpData
return tmpData
def updatePreview(self):
self.parseData(preview=True)
self.refreshDataTable()
def doParse(self):
self.done(1)
def updateModel(self):
if self.headerCheckBox.isChecked():
self.datatableModel = DataModel(self.data, True)
else:
self.datatableModel = DataModel(self.data, False)
def refreshDataTable(self):
if not self.data:
return
self.updateModel()
self.datatable.setModel(self.datatableModel)
self.datatable.horizontalHeader().setVisible(True)
self.datatable.verticalHeader().setVisible(True)
self.datatable.setShowGrid(True)
self.datatable.resizeColumnsToContents()
self.datatable.resizeRowsToContents()
headers = copy.copy(self.datatableModel.getHeader())
headers.insert(0,None)
if headers != self.header:
self.updateHeaderCombo(self.statusCombo, headers)
self.updateHeaderCombo(self.gazexCombo, headers)
self.updateHeaderCombo(self.gazeyCombo, headers)
self.updateHeaderCombo(self.timestampCombo, headers)
self.updateHeaderCombo(self.trialCombo, headers)
self.updateHeaderCombo(self.columnFilterComboBox, headers)
self.header = headers
def updateHeaderCombo(self, combobox, headers):
combobox.clear()
for i,v in enumerate(headers):
if i == 0:
combobox.addItem(v,None)
else:
combobox.addItem(v,i-1)
def getSegments(self, status_cb=None):
segments = []
tidx = self.trialCombo.currentIndex()-1
sidx = self.statusCombo.currentIndex()-1
data = self.datatableModel.getData()
header = self.datatableModel.getHeader()
current = [header]
status = []
line = 0
lines = len(data)
trial = 1
if tidx != -1:
trial = data[line][tidx]
while line < lines:
if tidx != -1 and data[line][tidx] != trial:
segments.append({'trial': trial, 'data': current, 'status': status})
current = [header]
status = []
trial = data[line][tidx]
if sidx == -1:
status.append(True)
else:
status.append(self.comparisonCombo.itemData(self.comparisonCombo.currentIndex())(float(data[line][sidx]),float(self.comparisonValue.text())))
current.append(data[line])
line += 1
segments.append({'trial': trial, 'data': current, 'status': status})
return {self.filename: {'segments': segments, 'firstRowIsHeader': self.headerCheckBox.isChecked()}}
import numpy as np
from datamodel import DataModel
if __name__=='__main__':
if False:
from tetrahedral import ChiralTetrahedral as Group
## from tetrahedral import Null as Group
## from octahedral import Tetrahedral as Group
## from octahedral import Null as Group
from octahedral import Pyritohedral as Group
## from octahedral import ChiralOctahedral as Group
## from octahedral import Tetrahedral as Group
## from octahedral import Origin as Group
## from dihedral import ChiralDihedral as Group
## from icosahedral import ChiralIcosahedral as Group
## from icosahedral import Icosahedral as Group
dm = DataModel(Group())
else:
## dm = DataModel.load(r'C:\Users\Eelco\Dropbox\Escheresque\examples\fishes.sch')
## dm = DataModel.load(r'C:\Users\Eelco\Dropbox\Escheresque\examples\angles_and_demons.sch')
dm = DataModel.load(r'C:\Users\Eelco Hoogendoorn\Dropbox\Escheresque\code\v2\test.sch')
print dm.edges
def updateModel(self):
if self.headerCheckBox.isChecked():
self.datatableModel = DataModel(self.data, True)
else:
self.datatableModel = DataModel(self.data, False)
def score(properties,
ts,
nRules,
filter_name=None,
group='Image',
show_results=False,
results_table=None,
overwrite=False):
'''
Trains a Classifier on a training set and scores the experiment
returns the table of scores as a numpy array.
properties -- Properties instance
ts -- TrainingSet instance
nRules -- number of rules to use
filter_name -- name of a filter to use from the properties file
group -- name of a group to use from the properties file
show_results -- whether or not to show the results in TableViewer
results_table -- table name to save results to or None.
'''
p = properties
db = DBConnect.getInstance()
dm = DataModel.getInstance()
if group == None:
group = 'Image'
if results_table:
if db.table_exists(results_table) and not overwrite:
print 'Table "%s" already exists. Delete this table before running scoreall.' % (
results_table)
return None
print ''
print 'properties: ', properties
print 'training set: ', ts
print '# rules: ', nRules
print 'filter: ', filter_name
print 'grouping by: ', group
print 'show results: ', show_results
print 'results table: ', results_table
print 'overwrite: ', overwrite
print ''
nClasses = len(ts.labels)
nKeyCols = len(image_key_columns())
assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s' % (
nRules, )
assert filter_name in p._filters.keys() + [
None
], 'Filter %s not found in properties file. Valid filters are: %s' % (
filter_name,
','.join(p._filters.keys()),
)
assert group in p._groups.keys() + [
'Image'
], 'Group %s not found in properties file. Valid groups are: %s' % (
group,
','.join(p._groups.keys()),
)
output = StringIO()
logging.info('Training classifier with %s rules...' % nRules)
t0 = time()
weaklearners = fastgentleboostingmulticlass.train(ts.colnames, nRules,
ts.label_matrix,
ts.values, output)
logging.info('Training done in %f seconds' % (time() - t0))
logging.info('Computing per-image class counts...')
t0 = time()
def update(frac):
logging.info('%d%% ' % (frac * 100., ))
keysAndCounts = multiclasssql.PerImageCounts(weaklearners,
filter_name=(filter_name
or None),
cb=update)
keysAndCounts.sort()
logging.info('Counts found in %f seconds' % (time() - t0))
if not keysAndCounts:
logging.error(
'No images are in filter "%s". Please check the filter definition in your properties file.'
% (filter_name))
raise Exception(
'No images are in filter "%s". Please check the filter definition in your properties file.'
% (filter_name))
# AGGREGATE PER_IMAGE COUNTS TO GROUPS IF NOT GROUPING BY IMAGE
if group != 'Image':
logging.info('Grouping %s counts by %s...' % (p.object_name[0], group))
t0 = time()
imData = {}
for row in keysAndCounts:
key = tuple(row[:nKeyCols])
imData[key] = np.array([float(v) for v in row[nKeyCols:]])
groupedKeysAndCounts = np.array([
list(k) + vals.tolist()
for k, vals in dm.SumToGroup(imData, group).items()
],
dtype=object)
nKeyCols = len(dm.GetGroupColumnNames(group))
logging.info('Grouping done in %f seconds' % (time() - t0))
else:
groupedKeysAndCounts = np.array(keysAndCounts, dtype=object)
# FIT THE BETA BINOMIAL
logging.info('Fitting beta binomial distribution to data...')
counts = groupedKeysAndCounts[:, -nClasses:]
alpha, converged = polyafit.fit_betabinom_minka_alternating(counts)
logging.info(' alpha = %s converged = %s' % (alpha, converged))
logging.info(' alpha/Sum(alpha) = %s' % ([a / sum(alpha)
for a in alpha]))
# CONSTRUCT ARRAY OF TABLE DATA
logging.info('Computing enrichment scores for each group...')
t0 = time()
tableData = []
for i, row in enumerate(groupedKeysAndCounts):
# Start this row with the group key:
tableRow = list(row[:nKeyCols])
if group != 'Image':
tableRow += [
len(dm.GetImagesInGroup(group, tuple(row[:nKeyCols])))
]
# Append the counts:
countsRow = [int(v) for v in row[nKeyCols:nKeyCols + nClasses]]
tableRow += [sum(countsRow)]
tableRow += countsRow
if p.area_scoring_column is not None:
# Append the areas
countsRow = [int(v) for v in row[-nClasses:]]
tableRow += [sum(countsRow)]
tableRow += countsRow
# Append the scores:
# compute enrichment probabilities of each class for this image OR group
scores = np.array(dirichletintegrate.score(alpha, np.array(countsRow)))
# clamp to [0,1] to
scores[scores > 1.] = 1.
scores[scores < 0.] = 0.
tableRow += scores.tolist()
# Append the logit scores:
# Special case: only calculate logit of "positives" for 2-classes
if nClasses == 2:
tableRow += [np.log10(scores[0]) - (np.log10(1 - scores[0]))
] # compute logit of each probability
else:
tableRow += [
np.log10(score) - (np.log10(1 - score)) for score in scores
] # compute logit of each probability
tableData.append(tableRow)
tableData = np.array(tableData, dtype=object)
logging.info('Enrichments computed in %f seconds' % (time() - t0))
# CREATE COLUMN LABELS LIST
# if grouping isn't per-image, then get the group key column names.
if group != 'Image':
colnames = dm.GetGroupColumnNames(group)
else:
colnames = list(image_key_columns())
# record the column indices for the keys
key_col_indices = [i for i in range(len(colnames))]
if group != 'Image':
colnames += ['Number_of_Images']
colnames += ['Total_%s_Count' % (p.object_name[0].capitalize())]
for i in xrange(nClasses):
colnames += [
'%s_%s_Count' %
(ts.labels[i].capitalize(), p.object_name[0].capitalize())
]
if p.area_scoring_column is not None:
colnames += ['Total_%s_Area' % (p.object_name[0].capitalize())]
for i in xrange(nClasses):
colnames += [
'%s_%s_Area' %
(ts.labels[i].capitalize(), p.object_name[0].capitalize())
]
for i in xrange(nClasses):
colnames += ['pEnriched_%s' % (ts.labels[i])]
if nClasses == 2:
colnames += ['Enriched_Score_%s' % (ts.labels[0])]
else:
for i in xrange(nClasses):
colnames += ['Enriched_Score_%s' % (ts.labels[i])]
title = results_table or "Enrichments_per_%s" % (group, )
if filter_name:
title += "_filtered_by_%s" % (filter_name, )
title += ' (%s)' % (os.path.split(p._filename)[1])
if results_table:
print 'Creating table %s' % (results_table)
success = db.CreateTableFromData(tableData,
colnames,
results_table,
temporary=False)
if not success:
print 'Failed to create results table :('
if show_results:
import tableviewer
tableview = tableviewer.TableViewer(None, title=title)
if results_table and overwrite:
tableview.load_db_table(results_table)
else:
tableview.table_from_array(tableData, colnames, group,
key_col_indices)
tableview.set_fitted_col_widths()
tableview.Show()
return tableData
def score(properties, ts, nRules, filter_name=None, group='Image',
show_results=False, results_table=None, overwrite=False):
'''
Trains a Classifier on a training set and scores the experiment
returns the table of scores as a numpy array.
properties -- Properties instance
ts -- TrainingSet instance
nRules -- number of rules to use
filter_name -- name of a filter to use from the properties file
group -- name of a group to use from the properties file
show_results -- whether or not to show the results in TableViewer
results_table -- table name to save results to or None.
'''
p = properties
db = DBConnect.getInstance()
dm = DataModel.getInstance()
if group == None:
group = 'Image'
if results_table:
if db.table_exists(results_table) and not overwrite:
print 'Table "%s" already exists. Delete this table before running scoreall.'%(results_table)
return None
print ''
print 'properties: ', properties
print 'training set: ', ts
print '# rules: ', nRules
print 'filter: ', filter_name
print 'grouping by: ', group
print 'show results: ', show_results
print 'results table: ', results_table
print 'overwrite: ', overwrite
print ''
nClasses = len(ts.labels)
nKeyCols = len(image_key_columns())
assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s'%(nRules,)
assert filter_name in p._filters.keys()+[None], 'Filter %s not found in properties file. Valid filters are: %s'%(filter_name, ','.join(p._filters.keys()),)
assert group in p._groups.keys()+['Image'], 'Group %s not found in properties file. Valid groups are: %s'%(group, ','.join(p._groups.keys()),)
output = StringIO()
logging.info('Training classifier with %s rules...'%nRules)
t0 = time()
weaklearners = fastgentleboostingmulticlass.train(ts.colnames,
nRules, ts.label_matrix,
ts.values, output)
logging.info('Training done in %f seconds'%(time()-t0))
logging.info('Computing per-image class counts...')
t0 = time()
def update(frac):
logging.info('%d%% '%(frac*100.,))
keysAndCounts = multiclasssql.PerImageCounts(weaklearners, filter_name=(filter_name or None), cb=update)
keysAndCounts.sort()
logging.info('Counts found in %f seconds'%(time()-t0))
if not keysAndCounts:
logging.error('No images are in filter "%s". Please check the filter definition in your properties file.'%(filter_name))
raise Exception('No images are in filter "%s". Please check the filter definition in your properties file.'%(filter_name))
# AGGREGATE PER_IMAGE COUNTS TO GROUPS IF NOT GROUPING BY IMAGE
if group != 'Image':
logging.info('Grouping %s counts by %s...' % (p.object_name[0], group))
t0 = time()
imData = {}
for row in keysAndCounts:
key = tuple(row[:nKeyCols])
imData[key] = np.array([float(v) for v in row[nKeyCols:]])
groupedKeysAndCounts = np.array([list(k)+vals.tolist() for k, vals in dm.SumToGroup(imData, group).items()], dtype=object)
nKeyCols = len(dm.GetGroupColumnNames(group))
logging.info('Grouping done in %f seconds'%(time()-t0))
else:
groupedKeysAndCounts = np.array(keysAndCounts, dtype=object)
# FIT THE BETA BINOMIAL
logging.info('Fitting beta binomial distribution to data...')
counts = groupedKeysAndCounts[:,-nClasses:]
alpha, converged = polyafit.fit_betabinom_minka_alternating(counts)
logging.info(' alpha = %s converged = %s'%(alpha, converged))
logging.info(' alpha/Sum(alpha) = %s'%([a/sum(alpha) for a in alpha]))
# CONSTRUCT ARRAY OF TABLE DATA
logging.info('Computing enrichment scores for each group...')
t0 = time()
tableData = []
for i, row in enumerate(groupedKeysAndCounts):
# Start this row with the group key:
tableRow = list(row[:nKeyCols])
if group != 'Image':
tableRow += [len(dm.GetImagesInGroup(group, tuple(row[:nKeyCols])))]
# Append the counts:
countsRow = [int(v) for v in row[nKeyCols:nKeyCols+nClasses]]
tableRow += [sum(countsRow)]
tableRow += countsRow
if p.area_scoring_column is not None:
# Append the areas
countsRow = [int(v) for v in row[-nClasses:]]
tableRow += [sum(countsRow)]
tableRow += countsRow
# Append the scores:
# compute enrichment probabilities of each class for this image OR group
scores = np.array( dirichletintegrate.score(alpha, np.array(countsRow)) )
# clamp to [0,1] to
scores[scores>1.] = 1.
scores[scores<0.] = 0.
tableRow += scores.tolist()
# Append the logit scores:
# Special case: only calculate logit of "positives" for 2-classes
if nClasses==2:
tableRow += [np.log10(scores[0])-(np.log10(1-scores[0]))] # compute logit of each probability
else:
tableRow += [np.log10(score)-(np.log10(1-score)) for score in scores] # compute logit of each probability
tableData.append(tableRow)
tableData = np.array(tableData, dtype=object)
logging.info('Enrichments computed in %f seconds'%(time()-t0))
# CREATE COLUMN LABELS LIST
# if grouping isn't per-image, then get the group key column names.
if group != 'Image':
colnames = dm.GetGroupColumnNames(group)
else:
colnames = list(image_key_columns())
# record the column indices for the keys
key_col_indices = [i for i in range(len(colnames))]
if group != 'Image':
colnames += ['Number_of_Images']
colnames += ['Total_%s_Count'%(p.object_name[0].capitalize())]
for i in xrange(nClasses):
colnames += ['%s_%s_Count'%(ts.labels[i].capitalize(), p.object_name[0].capitalize())]
if p.area_scoring_column is not None:
colnames += ['Total_%s_Area'%(p.object_name[0].capitalize())]
for i in xrange(nClasses):
colnames += ['%s_%s_Area'%(ts.labels[i].capitalize(), p.object_name[0].capitalize())]
for i in xrange(nClasses):
colnames += ['pEnriched_%s'%(ts.labels[i])]
if nClasses==2:
colnames += ['Enriched_Score_%s'%(ts.labels[0])]
else:
for i in xrange(nClasses):
colnames += ['Enriched_Score_%s'%(ts.labels[i])]
title = results_table or "Enrichments_per_%s"%(group,)
if filter_name:
title += "_filtered_by_%s"%(filter_name,)
title += ' (%s)'%(os.path.split(p._filename)[1])
if results_table:
print 'Creating table %s'%(results_table)
success = db.CreateTableFromData(tableData, colnames, results_table, temporary=False)
if not success:
print 'Failed to create results table :('
if show_results:
import tableviewer
tableview = tableviewer.TableViewer(None, title=title)
if results_table and overwrite:
tableview.load_db_table(results_table)
else:
tableview.table_from_array(tableData, colnames, group, key_col_indices)
tableview.set_fitted_col_widths()
tableview.Show()
return tableData
def score_objects(properties,
ts,
gt,
nRules,
filter_name=None,
group='Image',
show_results=False,
results_table=None,
overwrite=False):
'''
Trains a Classifier on a training set and scores the experiment
returns the table of scores as a numpy array.
properties -- Properties instance
ts -- TrainingSet instance
gt -- Ground Truth instance
nRules -- number of rules to use
filter_name -- name of a filter to use from the properties file
group -- name of a group to use from the properties file
show_results -- whether or not to show the results in TableViewer
results_table -- table name to save results to or None.
'''
p = properties
#db = DBConnect.getInstance() ## Removed writing to db. Results_table should be 'None' anyway
dm = DataModel.getInstance()
#if group == None:
#group = 'Image'
if results_table:
if db.table_exists(results_table) and not overwrite:
print 'Table "%s" already exists. Delete this table before running scoreall.' % (
results_table)
return None
print ''
print 'properties: ', properties
print 'initial training set: ', ts
print 'ground truth training set: ', gt
print '# rules: ', nRules
print 'filter: ', filter_name
print 'grouping by: ', group
print 'show results: ', show_results
print 'results table: ', results_table
print 'overwrite: ', overwrite
print ''
nClasses = len(ts.labels)
nKeyCols = len(image_key_columns())
assert 200 > nRules > 0, '# of rules must be between 1 and 200. Value was %s' % (
nRules, )
assert filter_name in p._filters.keys() + [
None
], 'Filter %s not found in properties file. Valid filters are: %s' % (
filter_name,
','.join(p._filters.keys()),
)
assert group in p._groups.keys() + [
'Image', 'None'
], 'Group %s not found in properties file. Valid groups are: %s' % (
group,
','.join(p._groups.keys()),
)
output = StringIO()
logging.info('Training classifier with %s rules...' % nRules)
t0 = time()
weaklearners = fastgentleboostingmulticlass.train(ts.colnames, nRules,
ts.label_matrix,
ts.values, output)
logging.info('Training done in %f seconds' % (time() - t0))
t0 = time()
#def update(frac):
#logging.info('%d%% '%(frac*100.,))
## Score Ground Truth using established classifier
gt_predicted_scores = per_cell_scores(weaklearners, gt.values, gt.colnames)
#plt.hist(gt_predicted_scores)
#plt.show()
gt_predicted_signs = np.sign(gt_predicted_scores)
## Compare Ground Truth score signs with the actual ground truth values
numclasses = ts.labels.size
gt_actual_signs = gt.label_matrix[:, 0]
cm_unrotated = metrics.confusion_matrix(gt_actual_signs,
gt_predicted_signs)
## sklearn.metrics.confusion_matrix -- 2D confusion matrix is inverted from convention.
## https://github.com/scikit-learn/scikit-learn/issues/1664
cm = np.rot90(np.rot90(cm_unrotated))
fpr, sens, thresholds = metrics.roc_curve(gt_actual_signs,
gt_predicted_signs)
spec = 1 - fpr
s = np.sum(cm, axis=1)
percent = [100 * cm[i, i] / float(s[i]) for i in range(len(s))]
avg = np.mean(percent)
avgTotal = 100 * np.trace(cm) / float(np.sum(cm))
print 'accuracy = %f' % avgTotal
print 'Confusion Matrix = ... '
print cm
my_sens = cm[0, 0] / float(cm[0, 0] + cm[0, 1]) #TP/(TP+FN)
my_spec = cm[1, 1] / float(cm[1, 1] + cm[1, 0]) #TN/(TN+FP)
print 'My_Sensitivity = %f' % my_sens
print 'My_Specificity = %f' % my_spec
print 'Sensitivity = ...'
print sens
print 'Specificity = ...'
print spec
print 'Done calculating'
############
## Confusion Matrix code from here: http://stackoverflow.com/questions/5821125/how-to-plot-confusion-matrix-with-string-axis-rather-than-integer-in-python
conf_arr = cm
norm_conf = []
## This normalizes each *row* to the color map, but I chose to normalize the whole confusion matrix to the same scale
##for i in conf_arr:
##a = 0
##tmp_arr = []
##a = sum(i, 0)
##for j in i:
##tmp_arr.append(float(j)/float(a))
##norm_conf.append(tmp_arr)
norm_conf = conf_arr / float(np.max(conf_arr))
if DISPLAY_CONFUSION_MATRIX:
fig = plt.figure()
plt.clf()
ax = fig.add_subplot(111)
ax.set_aspect(1)
res = ax.imshow(np.array(norm_conf),
cmap=plt.cm.jet,
interpolation='nearest')
width = len(conf_arr)
height = len(conf_arr[0])
for x in xrange(width):
for y in xrange(height):
ax.annotate(str(conf_arr[x][y]),
xy=(y, x),
horizontalalignment='center',
verticalalignment='center')
cb = fig.colorbar(res)
#cb.set_cmap = [0,1]
if width == 2 and height == 2:
plt.xticks([0, 1], ['FP', 'TN'])
plt.yticks([0, 1], ['TP', 'FP'])
else:
alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
plt.xticks(range(width), alphabet[:width])
plt.yticks(range(height), alphabet[:height])
plt.show()
print 'Done'
# -*- coding: utf-8 -*-
from PyQt4.QtGui import QApplication
import os
import sys
from mainWindow import MainWindow
from datamodel import DataModel
from moses import Moses
if __name__ == "__main__":
app = QApplication(sys.argv)
workdir = os.path.join(os.path.join(os.path.expanduser('~'), 'mosesgui'))
if not os.path.exists(workdir):
os.makedirs(workdir)
dm = DataModel(filename=os.path.join(workdir, "models.sqlite"))
moses = Moses()
if not moses.detect():
sys.exit(1)
MainWindow = MainWindow(dm=dm, moses=moses, workdir=workdir)
MainWindow.setupUi(MainWindow)
MainWindow.show()
sys.exit(app.exec_())
