def addPredictionsToImages(conn, prediction, dsId, commentImages, tagSet):
"""
Add a comment to the dataset containing the prediction results.
@param commentImages If true add comment to individual images as well
as the dataset
@param tagSet If provided then tag images with the predicted label
"""
message = ""
dsComment = ""
tagMap = {}
if tagSet:
for tag in tagSet.listTagsInTagset():
tagMap[tag.getValue()] = tag
for r in prediction.individual_results:
c = formatPredResult(r)
imId = long(r.source_file)
if commentImages:
message += WndcharmStorage.addCommentTo(conn, c, "Image", imId)
im = conn.getObject("Image", imId)
dsComment += im.getName() + " " + c + "\n"
if tagMap:
tag = tagMap[r.predicted_class_name]._obj
message += WndcharmStorage.addTagTo(conn, tag, "Image", imId)
message += WndcharmStorage.addCommentTo(conn, dsComment, "Dataset", dsId)
return message
def loadClassifier(ctb, project):
tidF = WndcharmStorage.getAttachedTableFile(ctb.tcF, project)
tidW = WndcharmStorage.getAttachedTableFile(ctb.tcW, project)
tidL = WndcharmStorage.getAttachedTableFile(ctb.tcL, project)
if tidF is None or tidW is None or tidL is None:
raise Exception("Incomplete set of classifier tables: %s" % (tidF, tidW, tidL))
ctb.openTables(tidF, tidW, tidL)
version = unwrap(ctb.versiontag.getTextValue())
cls = ctb.loadClassifierTables()
# ids,trainClassIds,featureMatrix,featureNames,weights,classIds,classNames
trainFts = wndcharm.FeatureSet.FeatureSet_Discrete()
# if cls['classIds'] != sorted(cls['classIds']):
if cls["classIds"] != range(len(cls["classIds"])):
raise Exception("Incorrectly ordered class IDs")
trainFts.classnames_list = cls["classNames"]
# Objects should be in order of increasing class ID, this makes rebuilding
# data_list much faster
nclasses = len(cls["classNames"])
classCounts = [0] * nclasses
cprev = -1
for c in cls["trainClassIds"]:
if c < cprev:
raise Exception("Incorrectly ordered training class feature data")
cprev = c
classCounts[c] += 1
trainFts.classsizes_list = classCounts
trainFts.feature_vector_version = version
classFts = [[] for n in xrange(len(cls["classNames"]))]
p = 0
for i in xrange(nclasses):
classFts[i] = numpy.array(cls["featureMatrix"][p : (p + classCounts[i])])
p += classCounts[i]
trainFts.data_list = classFts
trainFts.num_images = sum(classCounts)
trainFts.num_features = len(cls["featureNames"])
trainFts.num_classes = nclasses
trainFts.featurenames_list = cls["featureNames"]
trainFts.imagenames_list = [str(i) for i in cls["ids"]]
tmp = trainFts.ContiguousDataMatrix()
weights = wndcharm.FeatureSet.FisherFeatureWeights(
data_dict={"names": cls["featureNames"], "values": cls["weights"]}
)
return (trainFts, weights)
def predict(client, scriptParams):
message = ""
# for params with default values, we can get the value directly
projectId = scriptParams["Training_Project_ID"]
dataType = scriptParams["Data_Type"]
predictIds = scriptParams["IDs"]
commentImages = scriptParams["Comment_Images"]
tagImages = scriptParams["Tag_Images"]
contextName = scriptParams["Context_Name"]
tableNameIn = "/Wndcharm/" + contextName + WndcharmStorage.SMALLFEATURES_TABLE
tableNameF = "/Wndcharm/" + contextName + WndcharmStorage.CLASS_FEATURES_TABLE
tableNameW = "/Wndcharm/" + contextName + WndcharmStorage.CLASS_WEIGHTS_TABLE
tableNameL = "/Wndcharm/" + contextName + WndcharmStorage.CLASS_LABELS_TABLE
message += "tableNameIn:" + tableNameIn + "\n"
message += "tableNameF:" + tableNameF + "\n"
message += "tableNameW:" + tableNameW + "\n"
message += "tableNameL:" + tableNameL + "\n"
ftb = WndcharmStorage.FeatureTable(client, tableNameIn)
ctb = WndcharmStorage.ClassifierTables(client, tableNameF, tableNameW, tableNameL)
try:
message += "Loading classifier\n"
trainProject = ftb.conn.getObject("Project", projectId)
trainFts, weights = loadClassifier(ctb, trainProject)
classifierName = WndcharmStorage.CLASSIFIER_WNDCHARM_NAMESPACE
tagSet = WndcharmStorage.getClassifierTagSet(classifierName, trainProject.getName(), trainProject)
# Predict
message += "Predicting\n"
predDatasets = WndcharmStorage.datasetGenerator(ftb.conn, dataType, predictIds)
for ds in predDatasets:
message += "Predicting dataset id:%d\n" % ds.getId()
pred, msg = predictDataset(ftb, trainFts, ds, weights)
message += msg
message += addPredictionsToImages(ftb.conn, pred, ds.getId(), commentImages, tagSet)
except:
print message
raise
finally:
ftb.close()
ctb.close()
return message
def processImages(client, scriptParams):
message = ''
# for params with default values, we can get the value directly
dataType = scriptParams['Data_Type']
ids = scriptParams['IDs']
contextName = scriptParams['Context_Name']
tableName = '/Wndcharm/' + contextName + '/SmallFeatureSet.h5'
message += 'tableName:' + tableName + '\n'
ftb = WndcharmStorage.FeatureTable(client, tableName)
try:
# Get the datasets
objects, logMessage = script_utils.getObjects(ftb.conn, scriptParams)
message += logMessage
if not objects:
return message
datasets = WndcharmStorage.datasetGenerator(ftb.conn, dataType, ids)
for ds in datasets:
message += 'Processing dataset id:%d\n' % ds.getId()
msg = countCompleted(ftb, ds)
message += msg
except:
print message
raise
finally:
ftb.close()
return message
def addToFeatureSet(ftb, ds, fts, classId):
message = ""
tid = WndcharmStorage.getAttachedTableFile(ftb.tc, ds)
if tid:
if not ftb.openTable(tid):
return message + "\nERROR: Table not opened"
version = unwrap(ftb.versiontag.getTextValue())
message += "Opened table id:%d version:%s\n" % (tid, version)
else:
message += "ERROR: Table not found for Dataset id:%d" % ds.getId()
return message
# fts = wndcharm.FeatureSet.FeatureSet_Discrete({'num_images': 0})
for image in ds.listChildren():
imId = image.getId()
message += "\tProcessing features for image id:%d\n" % imId
# message += extractFeatures(tc, d, im = image) + '\n'
sig = wndcharm.FeatureSet.Signatures()
(sig.names, sig.values) = ftb.loadFeatures(imId)
# sig.source_file = image.getName()
sig.source_file = str(imId)
sig.version = version
fts.AddSignature(sig, classId)
fts.classnames_list[classId] = ds.getName()
return message
def crossValidate(ftb, project, featureThreshold, imagesOnly, numSplits):
message = ''
fullSet = FeatureSet_Discrete()
fullSet.source_path = project.getName()
classId = 0
for ds in project.listChildren():
message += 'Processing dataset id:%d\n' % ds.getId()
message += addToFeatureSet(ftb, ds, fullSet, classId, imagesOnly)
classId += 1
tmp = fullSet.ContiguousDataMatrix()
experiment = DiscreteClassificationExperimentResult(training_set=fullSet)
for i in range(numSplits):
trainSet, testSet = fullSet.Split()
trainSet.Normalize()
testSet.Normalize(trainSet)
weights = FisherFeatureWeights.NewFromFeatureSet(trainSet)
nFeatures = ceil(len(weights.names) * featureThreshold)
message += 'Selecting top %d features\n' % nFeatures
weights = weights.Threshold(nFeatures)
trainSet = reduceFeatures(trainSet, weights)
reducedTestSet = reduceFeatures(testSet, weights)
reducedTrainSet = reduceFeatures(trainSet, weights)
batchResult = DiscreteBatchClassificationResult.New(
reducedTrainSet, reducedTestSet, weights, batch_number=i)
experiment.individual_results.append(batchResult)
out = StringIO()
experiment.Print(output_stream=out)
experiment.PerSampleStatistics(output_stream=out)
pid = project.getId()
WndcharmStorage.addTextFileAnnotationTo(
ftb.conn, out.getvalue(), 'Project', pid,
'Wndcharm_Cross_Validation_Results.txt',
'Wndcharm Cross Validation Results for Project:%d' % pid)
message += 'Attached cross-validation results\n'
#return experiment
return message
def removeAnnotations(conn, obj, rmTables, rmComments, unlinkTags, rmTagsets):
"""
Remove annotations that are in one of the Wndcharm namespaces
"""
message = ''
rmIds = []
for ann in obj.listAnnotations():
if ann.getNs() == WndcharmStorage.WNDCHARM_NAMESPACE:
if rmTables and isinstance(ann, FileAnnotationWrapper):
# Need to remove version annotation on the OriginalFile
# otherwise delete will fail
WndcharmStorage.unlinkAnnotations(conn, ann.getFile())
message += ('Checking for annotations on file id:%d\n' %
ann.getFile().getId())
rmIds.append(ann.getId())
if rmComments and isinstance(ann, CommentAnnotationWrapper):
rmIds.append(ann.getId())
deleteObjects(conn, 'Annotation', rmIds)
message += 'Removed annotations:%s from %s id:%d\n' % \
(rmIds, obj.OMERO_CLASS, obj.getId())
if unlinkTags:
message += removeTagAnnotations(conn, obj)
if rmTagsets:
message += removeTags(conn, obj)
try:
# Keep recursing until listChildren not implemented
for ch in obj.listChildren():
message += removeAnnotations(
conn, ch, rmTables, rmComments, unlinkTags, rmTagsets)
except NotImplementedError:
pass
return message
def createWeights(ftb, ctb, project, featureThreshold, imagesOnly):
# Build the classifier (basically a set of weights)
message = ''
trainFts = wndcharm.FeatureSet.FeatureSet_Discrete()
classId = 0
for ds in project.listChildren():
message += 'Processing dataset id:%d\n' % ds.getId()
message += addToFeatureSet(ftb, ds, trainFts, classId, imagesOnly)
classId += 1
tmp = trainFts.ContiguousDataMatrix()
weights = wndcharm.FeatureSet.FisherFeatureWeights.NewFromFeatureSet(trainFts)
if featureThreshold < 1.0:
nFeatures = ceil(len(weights.names) * featureThreshold)
message += 'Selecting top %d features\n' % nFeatures
weights = weights.Threshold(nFeatures)
trainFts = reduceFeatures(trainFts, weights)
version = trainFts.feature_vector_version
# Save the features, weights and classes to tables
# TODO:Delete existing tables
#if getProjectTableFile(tcOutF, tcF.tableName, proj):
ctb.createClassifierTables(weights.names, version)
message += 'Created classifier tables ids: %d %d %d version:%s\n' % (
ctb.tcF.tableId, ctb.tcW.tableId, ctb.tcL.tableId, version)
# We've (ab)used imagenames_list to hold the image ids
ids = [long(a) for b in trainFts.imagenames_list for a in b]
classIds = [a for b in [[i] * len(z) for i, z in izip(xrange(
len(trainFts.imagenames_list)), trainFts.imagenames_list)]
for a in b]
featureMatrix = trainFts.data_matrix
featureNames = weights.names
featureWeights = weights.values
classNames = trainFts.classnames_list
ctb.saveClassifierTables(ids, classIds, featureMatrix,
featureNames, featureWeights, classNames)
WndcharmStorage.addFileAnnotationTo(ctb.tcF, project)
WndcharmStorage.addFileAnnotationTo(ctb.tcW, project)
WndcharmStorage.addFileAnnotationTo(ctb.tcL, project)
message += 'Saved classifier\n'
classifierName = WndcharmStorage.CLASSIFIER_WNDCHARM_NAMESPACE
ns = WndcharmStorage.createClassifierTagSet(
ctb.tcL.conn, classifierName, project.getName(), classNames, project)
message += 'Created tagset: %s\n' % ns
return trainFts, weights, message
def processImages(client, scriptParams):
message = ''
# for params with default values, we can get the value directly
dataType = scriptParams['Data_Type']
ids = scriptParams['IDs']
contextName = scriptParams['Context_Name']
newOnly = scriptParams['New_Images_Only']
tableName = '/Wndcharm/' + contextName + '/SmallFeatureSet.h5'
message += 'tableName:' + tableName + '\n'
ftb = WndcharmStorage.FeatureTable(client, tableName)
try:
nimages = 0
# Get the datasets
objects, logMessage = script_utils.getObjects(ftb.conn, scriptParams)
message += logMessage
if not objects:
return message
datasets = list(WndcharmStorage.datasetGenerator(
ftb.conn, dataType, ids))
good, chNames, msg = checkChannels(datasets)
message += msg
if not good:
raise omero.ServerError(
'Channel check failed, ' +
'all images must have the same channels: %s' % message)
for d in datasets:
message += 'Processing dataset id:%d\n' % d.getId()
for image in d.listChildren():
message += 'Processing image id:%d\n' % image.getId()
msg = extractFeatures(ftb, d, newOnly, chNames, im=image)
message += msg + '\n'
except:
print message
raise
finally:
ftb.close()
return message
def addToFeatureSet(ftb, ds, fts, classId, imagesOnly):
message = ''
tid = WndcharmStorage.getAttachedTableFile(ftb.tc, ds)
if tid:
if not ftb.openTable(tid):
return message + '\nERROR: Table not opened'
version = unwrap(ftb.versiontag.getTextValue())
message += 'Opened table id:%d version:%s\n' % (tid, version)
else:
message += 'ERROR: Table not found for Dataset id:%d' % ds.getId()
return message
#fts = wndcharm.FeatureSet.FeatureSet_Discrete({'num_images': 0})
if imagesOnly:
for image in ds.listChildren():
imId = image.getId()
message += '\tProcessing features for image id:%d\n' % imId
sig = wndcharm.FeatureSet.Signatures()
(sig.names, sig.values) = ftb.loadFeatures(imId)
sig.source_file = str(imId)
sig.version = version
fts.AddSignature(sig, classId)
else:
names, values, ids = ftb.bulkLoadFeatures()
message += '\tProcessing all features for dataset id:%d\n' % ds.getId()
for imId, vals in izip(ids, values):
sig = wndcharm.FeatureSet.Signatures()
sig.names = names
sig.values = vals
sig.source_file = str(imId)
sig.version = version
fts.AddSignature(sig, classId)
fts.classnames_list[classId] = ds.getName()
return message
def countCompleted(ftb, ds):
message = ''
tc = ftb.tc
imIds = [im.getId() for im in ds.listChildren()]
tid = WndcharmStorage.getAttachedTableFile(ftb.tc, ds)
if tid is None:
message += 'Image feature status PRESENT:%d ABSENT:%d\n' % \
(0, len(imIds))
return message
if not ftb.openTable(tid):
message += 'ERROR: Table not opened\n'
message += 'Image feature status UNKNOWN:%d\n' % len(imIds)
return message
message += 'Opened table id:%d\n' % tid
d = tc.chunkedRead([0], 0, tc.table.getNumberOfRows(), 100)
ftImIds = d.columns[0].values
matchedIds = set(imIds).intersection(ftImIds)
message += 'Image feature status PRESENT:%d ABSENT:%d\n' % \
(len(matchedIds), len(imIds) - len(matchedIds))
return message
def extractFeatures(ftb, ds, newOnly, chNames, imageId = None, im = None):
message = ''
tc = ftb.tc
# dataset must be explicitly provided because an image can be linked to
# multiple datasets in which case im.getDataset() doesn't work
if not im:
if not imageId:
#raise Exception('No input image')
raise omero.ServerError('No input image')
im = ftb.conn.getObject('Image', imageId)
if not im:
return 'Image id:%d not found\n' % imageId
else:
imageId = im.getId()
tid = WndcharmStorage.getAttachedTableFile(ftb.tc, ds)
if tid:
if not ftb.openTable(tid):
return message + '\nERROR: Table not opened\n'
version = unwrap(ftb.versiontag.getTextValue())
# version seems to be in unicode
message += 'Opened table id:%d version:%s\n' % (tid, str(version))
if newOnly and ftb.tableContainsId(imageId):
return message + 'Image id:%d features already in table' % imageId
# FIXME: default is convert multichannel to greyscale unless user input
# Calculate features for an image channel
# Optionally prepend the channel label to each feature name and combine
ftall = None
for c in xrange( len( chNames ) ):
wndcharm_matrix = PyImageMatrix()
wndcharm_matrix.allocate( im.getSizeX(), im.getSizeY() )
numpy_matrix = wndcharm_matrix.as_ndarray()
numpy_matrix[:] = im.getPrimaryPixels().getPlane(theZ=0,theC=c,theT=0)
feature_plan = wndcharm.StdFeatureComputationPlans.getFeatureSet();
options = "" # This is where you can tell wnd-charm to normalize pixel intensities,
# take ROIs etc. ... leave blank for now.
ft = Signatures.NewFromFeatureComputationPlan( wndcharm_matrix, feature_plan, options )
ft.names = [WndcharmStorage.insert_channel_name(
n, chNames[c]) for n in ft.names]
ft.source_path = im.getName()
if not ftall:
ftall = ft
else:
ftall.names += ft.names
ftall.values += ft.values
# Save the features to a table
if not tid:
ftb.createTable(ftall.names, ft.version)
version = unwrap(ftb.versiontag.getTextValue())
message += 'Created new table id:%d version:%s\n' % (
ftb.tc.tableId, version)
message += WndcharmStorage.addFileAnnotationTo(tc, ds)
if version != ft.version:
return message + 'Incompatible version: Stored=%s Calculated=%s' % (
version, ft.version)
ftb.saveFeatures(imageId, ftall)
return message + 'Extracted features from Image id:%d\n' % imageId