def main(opts):
testFiles = fileUtils.genfilelist(opts.testFileDir)
classFiles = fileUtils.genfilelist(opts.modelDir)
for testFile in testFiles:
predict = computeLabel(testFile, classFiles)
print('the label for given test file {} is: {}'.format(testFile, predict))
def loadModels(self, modelDir, NUM_CLASS):
cnnDict = defaultdict()
lstmDict = defaultdict()
saeDict = defaultdict()
fList = fileUtils.genfilelist(modelDir)
for fpath in fList:
modelName, index = differenciate(fpath)
if -1 == index:
continue
tmpDict = defaultdict()
self.opts.model = modelName
model, params = chooseModel(self.opts,
True,
NUM_CLASS=NUM_CLASS,
modelPath=fpath)
if 'cnn' == modelName:
cnnDict[index] = [model, params]
elif 'cudnnLstm' == modelName:
lstmDict[index] = [model, params]
elif 'sae' == modelName:
saeDict[index] = [model, params]
if not len(cnnDict.keys()) == len(saeDict.keys()) == len(
lstmDict.keys()) == self.nFold:
raise ValueError(
'models number for testing is not equal to nFold number: ',
self.nFold)
return cnnDict, lstmDict, saeDict
def test(testList, modelFileDir):
classFiles = fileUtils.genfilelist(modelFileDir)
predictions = []
for testfile in testList:
predict = computeLabel(testfile, classFiles)
predictions.append(predict)
return predictions
def genFileList(droot):
subDirs = os.listdir(droot)
fList = []
for subDir in subDirs:
dpath = os.path.join(droot, subDir)
tmpList = fileUtils.genfilelist(dpath)
fList.extend(tmpList)
return fList
def computeAllFeature(dpath):
fileList = fileUtils.genfilelist(dpath)
allFeatures = []
for fpath in fileList:
tmpFeat = computeFeature(fpath)
allFeatures.append(tmpFeat)
return np.array(allFeatures)
def loadData(dataDir, opts):
subDirs = fileUtils.getSubDirs(dataDir)
tmpDataList = []
tmpLabelList = []
fList = []
dirRoot = os.path.abspath(dataDir)
for subDir in subDirs:
dirpath = os.path.join(dirRoot, subDir)
tmpList = fileUtils.genfilelist(dirpath)
fList.extend(tmpList)
"""
for item in fList.copy():
tmp = os.path.basename(item)
cname = testByJaccard.getLabel(tmp)
if cname == 'what_are_the_most_popular_books_this_week':
fList.remove(item)
"""
labelMap = getLabelMap(fList)
totalNum = len(fList)
count = 1
for fp in fList:
if fileIsEmpty(fp):
print('skip empty file {}'.format(fp))
continue
tmpData = getFeature(fp, opts)
tmpDataList.append(tmpData)
tmpLabel = mapLabel(fp, labelMap)
tmpLabelList.append(tmpLabel)
print('\r now reading {:d}/{:d}'.format(count, totalNum), end='')
count = count + 1
allData = np.array(tmpDataList)
allLabel = np.array(tmpLabelList)
return allData, allLabel, labelMap
def loadTrainData(dataDir):
fList = fileUtils.genfilelist(dataDir)