def _getInstances(self, classAttr):
# create attributes
self.classAttr = classAttr
attName2Obj = {}
attVector = FastVector()
for attName in self.numericAttributes:
attr = Attribute(attName)
attVector.addElement(attr)
attName2Obj[attName] = attr
for (attName, domain) in self.attName2Domain.iteritems():
vDomain = FastVector(len(domain))
for v in domain:
#print v
vDomain.addElement(String(str(v)))
attr = Attribute(attName, vDomain)
attVector.addElement(attr)
attName2Obj[attName] = attr
self.attName2Obj = attName2Obj
# create Instances object
instances = Instances("instances", attVector, len(self.instances))
for i in self.instances:
inst = self._makeInstance(i)
instances.add(inst)
instances.setClass(attName2Obj[classAttr])
return instances
def readDataFromResultsTable(attributes, rt):
data = Instances("results", ArrayList(attributes), rt.size())
nrOfFeatures = len(attributes)
for i in range(0, rt.size()):
inst = DenseInstance(nrOfFeatures)
for j in range(0, nrOfFeatures):
value = rt.getValue(attributes[j].name(), i)
inst.setValue(attributes[j], value)
data.add(inst)
return data
def cluster(algorithm, filename, options=''):
reader = BufferedReader(FileReader(filename))
data = Instances(reader)
reader.close()
cl = algorithm()
cl.setOptions(options.split())
cl.buildClusterer(data)
returnData = []
for instance in data.enumerateInstances():
returnData.append(cl.clusterInstance(instance))
print returnData
def buildClassifier(self, instances):
"""
builds the ZeroR classifier with the given data
Parameter(s):
'instances' -- the data to build the classifier from
"""
self.getCapabilities().testWithFail(instances)
# remove instances with missing class
instances = Instances(instances)
instances.deleteWithMissingClass()
sumOfWeights = 0
self.__Class = instances.classAttribute()
self.__ClassValue = 0
self.__Counts = None
if (instances.classAttribute().isNumeric()):
self.__Counts = None
elif (instances.classAttribute().isNominal()):
self.__Counts = jarray.zeros(instances.numClasses(), 'd')
for i in range(len(self.__Counts)):
self.__Counts[i] = 1
sumOfWeights = instances.numClasses()
enu = instances.enumerateInstances()
while (enu.hasMoreElements()):
instance = enu.nextElement()
if (not instance.classIsMissing()):
if (instances.classAttribute().isNominal()):
self.__Counts[int(
instance.classValue())] += instance.weight()
else:
self.__ClassValue += instance.weight(
) * instance.classValue()
sumOfWeights += instance.weight()
if (instances.classAttribute().isNumeric()):
if (Utils.gr(sumOfWeights, 0)):
self.__ClassValue /= sumOfWeights
else:
self.__ClassValue = Utils.maxIndex(self.__Counts)
Utils.normalize(self.__Counts, sumOfWeights)
return
def classify(img, classifier, class_names, ops=None, distribution_class_index=-1):
""" img: a 2D RandomAccessibleInterval.
classifier: a WEKA Classifier instance, like SMO or FastRandomForest, etc. Any.
If it's a string, interprets it as a file path and attempts to deserialize
a previously saved trained classifier.
class_names: the list of names of each class to learn.
ops: the filter bank of ImgMath ops for the img.
distribution_class_index: defaults to -1, meaning return the class index for each pixel.
When larger than -1, it's interpreted as a class index, and
returns instead the floating-point value of each pixel in
the distribution of that particular class index. """
if type(classifier) == str:
classifier = SerializationHelper.read(classifier)
ops = ops if ops else filterBank(img)
attributes = ArrayList()
for i in xrange(len(ops)):
attributes.add(Attribute("attr-%i" % i))
#for name in classifier.attributeNames()[0][1]:
# attributes.add(Attribute(name))
attributes.add(Attribute("class", class_names))
info = Instances("structure", attributes, 1)
info.setClassIndex(len(attributes) -1)
opImgs = [compute(op).into(ArrayImgs.floats([img.dimension(0), img.dimension(1)])) for op in ops]
cs_opImgs = Views.collapse(Views.stack(opImgs))
result = ArrayImgs.floats([img.dimension(0), img.dimension(1)])
cr = result.cursor()
cop = Views.iterable(cs_opImgs).cursor()
while cr.hasNext():
tc = cop.next()
vector = array((tc.get(i).getRealDouble() for i in xrange(len(opImgs))), 'd')
vector += array([0], 'd')
di = DenseInstance(1.0, vector)
di.setDataset(info) # the list of attributes
if distribution_class_index > -1:
cr.next().setReal(classifier.distributionForInstance(di)[distribution_class_index])
else:
cr.next().setReal(classifier.classifyInstance(di))
return result
def createTrainingData(img, samples, class_names, n_samples=0, ops=None):
""" img: a 2D RandomAccessibleInterval.
samples: a sequence of long[] (or int numeric sequence or Localizable) and class_index pairs; can be a generator.
n_samples: optional, the number of samples (in case samples is e.g. a generator).
class_names: a list of class names, as many as different class_index.
ops: optional, the sequence of ImgMath ops to apply to the img, defaults to filterBank(img)
return an instance of WEKA Instances
"""
ops = ops if ops else filterBank(img)
if 0 == n_samples:
n_samples = len(samples)
# Define a WEKA Attribute for each feature (one for op in the filter bank, plus the class)
attribute_names = ["attr-%i" % (i+1) for i in xrange(len(ops))]
attributes = ArrayList()
for name in attribute_names:
attributes.add(Attribute(name))
# Add an attribute at the end for the classification classes
attributes.add(Attribute("class", class_names))
# Create the training data structure
training_data = Instances("training", attributes, n_samples)
training_data.setClassIndex(len(attributes) -1)
opImgs = [compute(op).into(ArrayImgs.floats([img.dimension(0), img.dimension(1)])) for op in ops]
ra = Views.collapse(Views.stack(opImgs)).randomAccess()
for position, class_index in samples:
ra.setPosition(position)
tc = ra.get()
vector = array((tc.get(i).getRealDouble() for i in xrange(len(opImgs))), 'd')
vector += array([class_index], 'd')
training_data.add(DenseInstance(1.0, vector))
return training_data
first parameter must be the ARFF file one wants to process with J48
Note: needs Weka 3.7.x to run (due to changes in the
weka.classifiers.Evaluation class)
"""
# check commandline parameters
if (not (len(sys.argv) == 2)):
print "Usage: UsingJ48Ext.py <ARFF-file>"
sys.exit()
# load data file
print "Loading data..."
file = FileReader(sys.argv[1])
data = Instances(file)
# set the class Index - the index of the dependent variable
data.setClassIndex(data.numAttributes() - 1)
# create the model
evaluation = Evaluation(data)
output = PlainText() # plain text output for predictions
output.setHeader(data)
buffer = StringBuffer() # buffer to use
output.setBuffer(buffer)
attRange = Range() # no additional attributes output
outputDistribution = Boolean(False) # we don't want distribution
j48 = J48()
j48.buildClassifier(data) # only a trained classifier can be evaluated
evaluation.evaluateModel(j48, data, [output, attRange, outputDistribution])
"""
Commandline parameter(s):
first parameter must be the ARFF file
"""
# check commandline parameters
if (not (len(sys.argv) == 2)):
print "Usage: supervised.py <ARFF-file>"
sys.exit()
# load data file
print "Loading data..."
datafile = FileReader(sys.argv[1] + ".arff")
data = Instances(datafile)
rand = Random() # seed from the system time
data.randomize(rand) # randomize data with number generator
# open output files
bufsize = 0
datafile = "data/plot/" + str(os.path.splitext(os.path.basename(__file__))[0]) + "_" + \
str(os.path.splitext(os.path.basename(sys.argv[1]))[0]) + "_rmse.csv"
file = open(datafile, 'w', bufsize) # open a file for rmse data
file.write("epochs,le,lm,kd,ball,cover\n")
logfile = "logs/" + str(os.path.splitext(os.path.basename(__file__))[0]) + "_" + \
str(os.path.splitext(os.path.basename(sys.argv[1]))[0]) + "_tunable.log"
log = open(logfile, 'w', bufsize) # open general log file
print "Usage: supervised.py <ARFF-file>"
sys.exit()
crossvalidate = sys.argv[2]
rand = Random() # seed from the system time
# load properties
p = Properties()
p.load(open('./ml.properties'))
# load data file
print "Loading data..."
trainfile = FileReader(sys.argv[1] + "-train.arff")
print "Loading " + sys.argv[1] + "-train.arff"
testfile = FileReader(sys.argv[1] + "-test.arff")
print "Loading " + sys.argv[1] + "-test.arff"
fulltrainset = Instances(trainfile)
fulltrainset.setClassIndex(fulltrainset.numAttributes() - 1)
testset = Instances(testfile)
testset.setClassIndex(testset.numAttributes() - 1)
# open output files
bufsize = 0
classifiername = str(os.path.splitext(os.path.basename(__file__))[0])
dataname = str(os.path.splitext(os.path.basename(sys.argv[1]))[0])
datafilelimit = "data/plot/" + classifiername + "_" + dataname + crossvalidate + "_instances.csv"
filelimit = open(datafilelimit, 'w', bufsize)
filelimit.write(
"instances,lineartest,lineartrain,polytest,polytrain,radialtest,radialtrain,sigmoidtest,sigmoidtrain\n"
)
logfile = "logs/" + classifiername + "_" + dataname + crossvalidate + ".log"
log = open(logfile, 'w', bufsize) # open general log file
import weka.core.Instances as Instances
import weka.classifiers.trees.J48 as J48
import weka.classifiers.Evaluation as Evaluation
import weka.core.Range as Range
import weka.classifiers.functions.MultilayerPerceptron as MultilayerPerceptron
import weka.core.SerializationHelper as SerializationHelper
# check commandline parameters
if (not (len(sys.argv) == 3)):
print "Usage: weka.py <ARFF-file>"
sys.exit()
file = FileReader(sys.argv[1])
file2 = FileReader(sys.argv[2])
data = Instances(file)
test = Instances(file2)
data.setClassIndex(data.numAttributes() - 1)
test.setClassIndex(test.numAttributes() - 1)
evaluation = Evaluation(data)
buffer = StringBuffer()
attRange = Range() # no additional attributes output
outputDistribution = Boolean(False) # we don't want distribution
nn = MultilayerPerceptron()
nn.buildClassifier(data) # only a trained classifier can be evaluated
#print evaluation.evaluateModel(nn, ['-t', sys.argv[1], '-T', sys.argv[2]])#;, [buffer, attRange, outputDistribution])
res = evaluation.evaluateModel(nn, test,
[buffer, attRange, outputDistribution])
f = open('predictions/' + data.relationName(), 'w')
for d in res:
# Define a WEKA Attribute for each feature (each op, 14 total plus the class)
attribute_names = ["attr-%i" % (i + 1) for i in xrange(14)]
attributes = ArrayList()
for name in attribute_names:
attributes.add(Attribute(name))
# Add an attribute at the end for the classification classes
attributes.add(
Attribute("class",
["membrane", "mit-boundary", "mit-inside", "cytoplasm"]))
# Create the training data structure
# which consists of 16 samples for each membrane training image rotation
# and 4 samples for each mitochondrial boundary image rotation
# and times 2 to then add examples of the other, non-membrane class
training_data = Instances(
"training", attributes,
(len(synth_imgs_membrane) * 16 + len(synth_imgs_mit_boundary) * 4) * 2)
training_data.setClassIndex(len(attributes) - 1)
def populateInstances(instances, synth_imgs, class_index, mins, maxs):
# Populate the training data: create the filter bank for each feature image
# by reading values from the interval defined by mins and maxs
target = ArrayImgs.floats([width, height])
interval = FinalInterval(mins, maxs)
n_samples = Intervals.numElements(interval)
for img in synth_imgs:
vectors = [zeros(len(attributes), 'd') for _ in xrange(n_samples)]
for k, op in enumerate(filterBank(img, sumType=DoubleType())):
imgOp = compute(op).into(target)
for i, v in enumerate(Views.interval(imgOp, interval)):
percentiles = []
for i in range(0, 100, step_size):
index = int(n * i / 100)
percentiles.append(thread_array[index]) # 0-99 percentile
percentiles.append(thread_array[n - 1]) # 100th percentile
f = open(''.join([directory, 'percentiles.txt']), 'w')
f.write(str(percentiles))
f.close()
#print str(percentiles)
## Data Distribution testing/Training
data = FileReader(data_file)
data = Instances(data)
data = Instances(data, 0, n - (n % folds))
n = n - (n % folds)
print data.numInstances()
len_fold = int(math.floor(n / folds))
folds_test = []
folds_train = []
for i in range(0, n + 1, len_fold)[:-1]:
folds_test.append(Instances(data, i, len_fold))
f = open(
''.join([
directory, ''.join(['fold_test_',
str(i / len_fold), '.arff'])
]), "w")
f.write(str(folds_test[-1]))
f.close()
"""
Commandline parameter(s):
first parameter must be the ARFF file
"""
# check commandline parameters
if (not (len(sys.argv) == 2)):
print "Usage: supervised.py <ARFF-file>"
sys.exit()
# load data file
print "Loading data..."
datafile = FileReader(sys.argv[1])
data = Instances(datafile)
rand = Random() # seed from the system time
data.randomize(rand) # randomize data with number generator
# open output files
bufsize = 0
dataname = str(os.path.splitext(os.path.basename(sys.argv[1]))[0])
# loop for different amounts of data with fixed test set
datasize = data.numInstances()
limit = (datasize * 2) / 3 # loop until we use 2/3 data as training set
testset = Instances(data, limit, datasize -
limit) # create training set using the last 1/3 of data
testset.setClassIndex(testset.numAttributes() - 1)
saver = ArffSaver()
