# bitformat for categoryData
preCategoryHeader, preCategoryData = bpModelDataPrcocessor.getPreProcessedFlattenCategoryData((categoryHeader, npCategoryData), mongoDataProcessor.getCategoryInfo())
np.savetxt('./Data/preCategoryData.csv',preCategoryData,header=','.join(preCategoryHeader).encode('utf-8'),delimiter=',', fmt='%d',comments='')
# PCA+KMeans for numericalData
PCAResultMap, transResultMap = bpModelDataPrcocessor.getAssociatedMapFromPCA((numericalHeader, zscoreNumericalData))
kmLists = bpModelDataPrcocessor.getKMeansListByCalculation((numericalHeader, zscoreNumericalData), (PCAResultMap, transResultMap), path='./Figures/')
bpModelDataPrcocessor.saveKMeansListToFile('./KMeansModel/', numericalHeader)
preNumericalHeader, preNumericalData = bpModelDataPrcocessor.getPreProcessedFlattenNumericalData((numericalHeader, zscoreNumericalData), dropTags=[])
np.savetxt('./Data/preNumericalData.csv',preNumericalData,header=','.join(preNumericalHeader).encode('utf-8'),delimiter=',', fmt='%d',comments='')
# ANN
bPModelTrainer = BPModelTrainer(
flattenCategoryHeader=preCategoryHeader,
flattenCategoryData=preCategoryData,
flattenNumericalHeader=preNumericalHeader,
flattenNumericalData=preNumericalData,
flattenTargetHeader=targetHeader,
flattenTargetData=targetData
)
# transport targetData to discrete value
bPModelTrainer.targetTransform()
# ANN train
bPModelTrainer.trainModel()
# validate
bPModelTrainer.validateModel()
# save
bPModelTrainer.saveModelToFile('Data/NNModel.xml')
with open('Data/PreProcessedFlattenNumericalData.csv') as f:
header = f.readline()
preNumericalHeader = header.split(',')
preNumericalData = np.genfromtxt('Data/PreProcessedFlattenNumericalData.csv', delimiter=',', skip_header=1, dtype=np.int)
with open('Data/TargetData.csv') as f:
header = f.readline()
targetHeader = header.split(',')
targetData = np.genfromtxt('Data/TargetData.csv', delimiter=',', skip_header=1, dtype=np.int)
bPModelTrainer = BPModelTrainer(
flattenCategoryHeader=preCategoryHeader,
flattenCategoryData=preCategoryData,
flattenNumericalHeader=preNumericalHeader,
flattenNumericalData=preNumericalData,
flattenTargetHeader=targetHeader,
flattenTargetData=targetData
)
# Step14: transform the target data to needed one
bPModelTrainer.targetTransform()
#Step15: generate the result distribution accroding to the tags
distMap = bPModelTrainer.generateRiskDistributionByRiskTags()
with open('Data/ResultDistribution.txt', 'w') as f:
outStr = ''
for item in distMap.items():
tagGroup, value = item
outStr += tagGroup + '\n'
for item in value.items():
(numericalHeader, zscoreNumericalData))
kmLists = bpModelDataPrcocessor.getKMeansListByCalculation(
(numericalHeader, zscoreNumericalData), (PCAResultMap, transResultMap),
path='./Figures/')
bpModelDataPrcocessor.saveKMeansListToFile('./KMeansModel/', numericalHeader)
preNumericalHeader, preNumericalData = bpModelDataPrcocessor.getPreProcessedFlattenNumericalData(
(numericalHeader, zscoreNumericalData), dropTags=[])
np.savetxt('./Data/preNumericalData.csv',
preNumericalData,
header=','.join(preNumericalHeader).encode('utf-8'),
delimiter=',',
fmt='%d',
comments='')
# ANN
bPModelTrainer = BPModelTrainer(flattenCategoryHeader=preCategoryHeader,
flattenCategoryData=preCategoryData,
flattenNumericalHeader=preNumericalHeader,
flattenNumericalData=preNumericalData,
flattenTargetHeader=targetHeader,
flattenTargetData=targetData)
# transport targetData to discrete value
bPModelTrainer.targetTransform()
# ANN train
bPModelTrainer.trainModel()
# validate
bPModelTrainer.validateModel()
# save
bPModelTrainer.saveModelToFile('Data/NNModel.xml')
