def gen_verilog_test_file_aux(data_num: int = 1,
fft_bit: int = 8,
dir: str = "原码/"):
"""
生成原码表示的测试verilog文件
:param data_num:
:param fft_bit:
:return:
"""
if not os.path.isdir(dir):
os.mkdir(dir)
d = DataHandler()
d.gen_data(data_num, fft_bit)
input_list, output_list = d.from_divide_list_to_complex()
input_f = open(os.path.join(dir, "补码/input_data.txt"), 'w')
output_f = open(os.path.join(dir, "补码/result_data.txt"), 'w')
input_test_f = open(os.path.join(dir, "input_test_data.txt"), 'w')
output_test_f = open(os.path.join(dir, "output_test_data.txt"), 'w')
for i in range(data_num):
input = complex_convert(input_list[i], 0, 9)
output = complex_convert(output_list[i], 3, 6)
input_real_bit, input_imag_bit = convert(input, 0, 9)
output_real_bit, output_imag_bit = convert(output, 3, 6)
for j in range(fft_bit):
input_test_f.writelines(str(input_list[i][j]) + '\n')
output_test_f.writelines(str(output_list[i][j]) + '\n')
input_f.writelines(input_real_bit[j] + " " + input_imag_bit[j] +
"\n")
output_f.writelines(output_real_bit[j] + " " + output_imag_bit[j] +
"\n")
input_f.close()
output_f.close()
output_test_f.close()
input_test_f.close()
def __init__(self,
model,
dataFunction,
trainSize=0.7,
preprocessed=True,
folderID=None,
nDataPoints=100000):
if model in "DecisionTree":
self.model = OwnDecisionTree
elif model in "RandomForest":
self.model = OwnRandomForest
elif model in "LinearRegression":
self.model = OwnLinearReg
elif model in "LassoLarsCV":
self.model = OwnLassoLarsCV
else:
print(f'{model} not found!')
exit(1)
self.nDataPoints = nDataPoints
self.preprocessed = preprocessed
self.trainSize = trainSize
self.modelName = self.model.__name__
self.dataFunction = dataFunction
self.dataUsedName = self.dataFunction.__name__.split("_")[1]
# Export path
self.savePath = f'{os.path.dirname(os.path.abspath(__file__))}/runs/{self.dataUsedName}_{self.trainSize}/{self.modelName}'
# Add folder with number that represents evaluation run
self.savePath = DataHandler.createDictPath(self.savePath, folderID,
False)
def __init__(self,
dataFunction,
trainSize=0.1,
feateng_steps=3,
featuresel_steps=5,
steps="",
folderID=None,
nDataPoints=100000):
self.trainSize = trainSize
self.dataFunction = dataFunction
self.dataUsedName = self.dataFunction.__name__.split("_")[1]
self.feateng_steps = feateng_steps
self.featuresel_steps = featuresel_steps
self.steps = steps
self.nDataPoints = nDataPoints
# Export path
self.savePath = f'{os.path.dirname(os.path.abspath(__file__))}/runs/{self.dataUsedName}_{self.trainSize}_{steps}'
# Add folder with number that represents evaluation run
self.savePath = DataHandler.createDictPath(self.savePath, folderID)
def __init__(self,
dataFunction,
generations=100,
popSize=100,
trainSize=0.7,
model: str = 'sklearn.tree.DecisionTreeRegressor',
preprocessed=True,
folderID=None,
nDataPoints=100000):
self.generations = generations
self.popSize = popSize
self.trainSize = trainSize
self.model = model
self.preprocessed = preprocessed
self.nDataPoints = nDataPoints
# For ease of use
if model in "DecisionTree":
self.model = 'sklearn.tree.DecisionTreeRegressor'
elif model in "RandomForest":
self.model = {
"sklearn.ensemble.RandomForestRegressor": {
'n_estimators': [10]
}
}
elif model in "LinearRegression":
self.model = "sklearn.linear_model.LinearRegression"
elif model in "LassoLarsCV":
#Settings from autofeat
self.model = {"sklearn.linear_model.LassoLarsCV": {'cv': [5]}}
if isinstance(self.model, str):
self.model = {self.model: {}}
self.modelName = list(self.model.keys())[0].split(".")[-1]
self.dataFunction = dataFunction
self.dataUsedName = self.dataFunction.__name__.split("_")[1]
# Export path
self.savePath = f'{os.path.dirname(os.path.abspath(__file__))}/runs/{self.dataUsedName}_{self.trainSize}/{self.modelName}_gen{self.generations}_pop{self.popSize}'
# Add folder with number that represents evaluation run
self.savePath = DataHandler.createDictPath(self.savePath, folderID)
type=int)
parser.add_argument('--featsel_runs',
help='Number of feature selection steps',
default=5,
type=int)
parser.add_argument('--folderID', help='ID for folder')
parser.add_argument('--nDataPoints',
type=int,
help='Reduce data to subsample size.',
default=100000)
args = parser.parse_args()
if len(sys.argv) > 1:
if "reg" in args.problem:
autofeatTransformer = AutofeatWrapper(
dataFunction=DataHandler.stringToMethod(args.data),
trainSize=args.trainSize,
feateng_steps=args.feateng_steps,
featuresel_steps=args.featsel_runs,
folderID=args.folderID,
nDataPoints=args.nDataPoints)
# User input 100 means try every combination
if args.feateng_steps == 100 and args.featsel_runs == 100:
maxEngSteps = 3
# dataset not feasible for categorical values with 8gb of RAM
if "rossmann" in args.data:
maxEngSteps = 2
for i in range(maxEngSteps):
for j in range(5):
autofeatTransformer.feateng_steps = i + 1
autofeatTransformer.featuresel_steps = j + 1
def startQueryData():
data = DataHandler()
data.start()
def checkAccuracy(generatedIndicies, image, filename):
truePositives = 0
falsePositives = 0
falseNegatives = 0
noduleCount = 0
nodulesFound = 0
#Generates the origin and spacing values for the image
handler = DataHandler()
filepath = "../Data/toPredict/" + filename + ".mhd"
_, origin, spacing = handler.load_itk_image(filepath)
#Generates the list of indicies that we need for the file
points = image[filename]
correctIndicies = {}
for point in points:
#Converts values in points to voxel and scales them correctly
worldCoords = np.asarray(
[float(point[0]),
float(point[1]),
float(point[2])])
#print(worldCoords)
voxelCoords = handler.worldToVoxelCoord(worldCoords, origin, spacing)
#print(voxelCoords)
#Scales for the resolution that we want
targetSpacing = np.array([1, .625, .625])
multiplier = spacing / targetSpacing
#print("-----")
#print(voxelCoords)
voxelCoords = voxelCoords * multiplier
#print(voxelCoords)
#print("-----")
valuesTuple = (tuple(voxelCoords), int(point[3]))
correctIndicies[valuesTuple] = 0
#Goes through each guessed index and figures out of it is false or not
for guess in generatedIndicies:
#Goes through each correct answer
isCorrect = False
for correct in correctIndicies:
#Guess coordinates you are judging based off of
guessZ = guess[0][0]
guessX = guess[0][1]
guessY = guess[0][2]
correctZ = correct[0][0]
correctX = correct[0][1]
correctY = correct[0][2]
if (guessZ - 5 < correctZ) and (guessZ + 5 > correctZ) and (
guessX - 8 < correctX) and (guessX + 8 > correctX) and (
guessY - 8 < correctY) and (guessY + 8 > correctY):
correctIndicies[correct] = 1
isCorrect = True
if isCorrect == False:
falsePositives = falsePositives + 1
#If the guess was not any correct answer then it is a false positive
#if not isCorrect:
# falsePositives += 1
for ind in correctIndicies:
if ind[1] == 1:
noduleCount = noduleCount + 1
if correctIndicies[ind] == 1:
nodulesFound = nodulesFound + 1
#If we found it
if correctIndicies[ind] == 1:
truePositives = truePositives + 1
if correctIndicies[ind] == 0:
falseNegatives = falseNegatives + 1
numPositives = len(correctIndicies)
return truePositives, numPositives, falsePositives, falseNegatives, noduleCount, nodulesFound
for ind in correctIndicies:
if ind[1] == 1:
noduleCount = noduleCount + 1
if correctIndicies[ind] == 1:
nodulesFound = nodulesFound + 1
#If we found it
if correctIndicies[ind] == 1:
truePositives = truePositives + 1
if correctIndicies[ind] == 0:
falseNegatives = falseNegatives + 1
numPositives = len(correctIndicies)
return truePositives, numPositives, falsePositives, falseNegatives, noduleCount, nodulesFound
handler = DataHandler()
FCN = FCN(2, "../Data/model/", "model/8")
imageNames, keptIndiciesList = getCandidates("../Data/toPredict/", FCN)
#print(keptIndiciesList)
generateStats(imageNames, keptIndiciesList)
'''
y , orig, spacing = handler.load_itk_image("../Data/data/1.3.6.1.4.1.14519.5.2.1.6279.6001.220596530836092324070084384692.mhd")
#pred = predFull(FCN, y, spacing, (1, 0.625, 0.625))
#print(pred.shape)
cand = np.load("../Data/fullresnorm5.npy")
print(cand.shape)
handler.save_slices(y, 10, "../Data/images/CTscan/")
xs, ys = handler.load_samples("../Data/sampless/subset{}/".format(i), (10,16,16,1))
print(xs.shape)
print(ys.shape)
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.optimizers import SGD
from keras.layers.convolutional import Conv3D
from keras.layers.convolutional import MaxPooling3D
from keras.utils import np_utils
from keras import backend as K
from keras.models import model_from_json
import random
import os
from Data import DataHandler
handler = DataHandler()
K.set_image_dim_ordering('th')
num_classes = 2
# Create the model
model = Sequential()
#Runs 64 (3,5,5) Kernels (features) over the image
#Run activation maps through relu to return 0 for negative values
#Constrains the Kernel to be size 3 at most?
#The output now has 32 activation maps, each for a different feature
def startQueryData():
data = DataHandler()
data.start()
def testData():
Data = DataHandler()
Data.start()
pass
visualiseBestSoFar(getPerformanceHistory_autosklearn("rossmann_0.7"), y_label="RMSE", x_label="Evaluation number", title="Auto-sklearn performance for Rossmann", saveName="autosklearn_rossmann")
visualiseBestSoFar(getPerformanceHistory_autosklearn("taxitrip_0.7"), y_label="RMSE", x_label="Evaluation Number", title="Autos-sklearn performance for TaxiTrip", saveName="autosklearn_taxitrip")
#%% AutofeatPlots
visualizeAutofeatPerformance("synthetic_0.7", default=syntheticPerformance, saveName="autofeat_synthetic", vmax=100)
visualizeAutofeatPerformance("rossmann_0.7", default=rossmannPerformance, saveName="autofeat_rossmann", vmax=30)
visualizeAutofeatPerformance("taxitrip_0.7", default=taxitripPerformance, saveName="autofeat_taxitrip", vmax=30)
#%% Total performance over all
visualizeTotalPerformance(getTotalPerformances("synthetic_0.7", default=syntheticPerformance), modelOrder, title="Framework performance for Synthetic", saveName="total_Synthetic", inPercent=False, plotKaggle=True, logScale=True, loc='upper right')
visualizeTotalPerformance(getTotalPerformances("rossmann_0.7", default=rossmannPerformance), modelOrder, title="Framework performance for Rossmann", saveName="total_rossmann", inPercent=False, plotKaggle=True, loc='upper right')
visualizeTotalPerformance(getTotalPerformances("taxitrip_0.7", default=taxitripPerformance), modelOrder, title="Framework performance for TaxiTrip", saveName="total_taxitrip", inPercent=False, plotKaggle=True, loc='upper right')
#%% plot target data distribution
_,_,targetValues_taxitrip_def,_ = DataHandler.getData_taxitrip(trainSize=0.9999)
_,_,targetValues_taxitrip,_ = DataHandler.getData_taxitrip(trainSize=0.9999, preprocessed=True)
fig, axs = plt.subplots(2,1, dpi=200)
axs[0].boxplot(targetValues_taxitrip_def, vert=False)
axs[0].set_title("Target distribution")
axs[0].set_yticklabels([""])
axs[0].set_xlabel("Seconds")
axs[1].boxplot(targetValues_taxitrip, vert=False)
axs[1].set_title(" Target distribution after removing outliers")
axs[1].set_yticklabels([""])
axs[1].set_xlabel("Seconds")
plt.tight_layout(rect=(0,0,1,0.6))