def run(self, className, outfilename, param, dsname, gtname, evalconfig):
try:
classifier = param['classifier']
gt = GroundTruth(classifier)
gt.load(gtname)
# force the GroundTruth class name to be the one specified by our project file, not
# the one in the original groundtruth file
gt.className = className
ds = DataSet()
ds.load(dsname)
# some points may have failed to be analyzed, remove those from the GroundTruth
pnames = ds.pointNames()
for pid in list(gt.keys()):
if pid not in pnames:
log.warning(
'Removing %s from GroundTruth as it could not be found in the merged dataset'
% pid)
del gt[pid]
trainerFun, trainingparam, newds = getTrainer(
classifier, param, ds)
# run all the evaluations specified in the evaluation config
for i, evalparam in enumerate(evalconfig):
# if we already ran this evaluation, no need to run it again...
resultFilename = outfilename + '_%d.result' % i
if exists(resultFilename):
log.info('File %s already exists. Skipping evaluation...' %
resultFilename)
continue
log.info(
'Running evaluation %d for: %s with classifier %s and dataset %s'
% (i, outfilename, param['classifier'],
param['preprocessing']))
log.info(' PID: %d, parameters: %s' %
(os.getpid(), json.dumps(param)))
# run evaluation
confusion = evaluateNfold(evalparam['nfold'], ds, gt,
trainerFun, **trainingparam)
# write evaluation params & result
with open(outfilename + '_%d.param' % i, 'w') as f:
yaml.dump({'model': param, 'evaluation': evalparam}, f)
confusion.save(resultFilename)
except Exception:
log.error(
'While doing evaluation with param = %s\nevaluation = %s' %
(param, evalconfig))
raise
def run(self, className, outfilename, param, dsname, gtname, evalconfig):
try:
classifier = param['classifier']
gt = GroundTruth(classifier)
gt.load(gtname)
# force the GroundTruth class name to be the one specified by our project file, not
# the one in the original groundtruth file
gt.className = className
ds = DataSet()
ds.load(dsname)
# some points may have failed to be analyzed, remove those from the GroundTruth
pnames = ds.pointNames()
for pid in list(gt.keys()):
if pid not in pnames:
log.warning('Removing %s from GroundTruth as it could not be found in the merged dataset' % pid)
del gt[pid]
trainerFun, trainingparam, newds = getTrainer(classifier, param, ds)
# run all the evaluations specified in the evaluation config
for i, evalparam in enumerate(evalconfig):
# if we already ran this evaluation, no need to run it again...
resultFilename = outfilename + '_%d.result' % i
if exists(resultFilename):
log.info('File %s already exists. Skipping evaluation...' % resultFilename)
continue
log.info('Running evaluation %d for: %s with classifier %s and dataset %s' % (i, outfilename,
param['classifier'],
param['preprocessing']))
log.info(' PID: %d, parameters: %s' % (os.getpid(), json.dumps(param)))
# run evaluation
confusion = evaluateNfold(evalparam['nfold'], ds, gt, trainerFun, **trainingparam)
# write evaluation params & result
with open(outfilename + '_%d.param' % i, 'w') as f:
yaml.dump({ 'model': param, 'evaluation': evalparam }, f)
confusion.save(resultFilename)
except Exception:
log.error('While doing evaluation with param = %s\nevaluation = %s' % (param, evalconfig))
raise
alpha = float(sys.argv[6])
filename = resultsdir + basename + '_simca_%d_%d_%.1f' % (
featureSet, coveredVar, alpha)
if os.path.exists(filename + '.result'):
print 'File exists. Skipping Test.'
else:
ds, groundTruth = loadData(datasetName, featureSet)
from classifier_SIMCA import train_SIMCA
confusion = evaluateNfold(10,
ds,
groundTruth,
train_SIMCA,
descriptorNames='*',
exclude=[],
alpha=alpha,
coveredVariance=coveredVar,
useBoundaryDistance=True)
print confusion.results()
writeResult(filename, confusion)
elif classifier == 'svm':
svmtype = sys.argv[5]
kernel = sys.argv[6]
cexp = float(sys.argv[7])
gammaexp = float(sys.argv[8])
filename = resultsdir + basename + '_svm_%d_%s_%s_%.1f_%.1f' % (
featureSet = int(sys.argv[4])
if classifier == 'simca':
coveredVar = int(sys.argv[5])
alpha = float(sys.argv[6])
filename = resultsdir + basename + '_simca_%d_%d_%.1f' % (featureSet, coveredVar, alpha)
if os.path.exists(filename + '.result'):
print 'File exists. Skipping Test.'
else:
ds, groundTruth = loadData(datasetName, featureSet)
from classifier_SIMCA import train_SIMCA
confusion = evaluateNfold(10, ds, groundTruth, train_SIMCA,
descriptorNames = '*', exclude = [],
alpha = alpha, coveredVariance = coveredVar,
useBoundaryDistance = True)
print confusion.results()
writeResult(filename, confusion)
elif classifier == 'svm':
svmtype = sys.argv[5]
kernel = sys.argv[6]
cexp = float(sys.argv[7])
gammaexp = float(sys.argv[8])
filename = resultsdir + basename + '_svm_%d_%s_%s_%.1f_%.1f' % (featureSet, svmtype, kernel, cexp, gammaexp)
if os.path.exists(filename + '.result'):
print 'File exists. Skipping Test.'
