def train_model(args): """ Trains the model with the input data specified in the CLI arguments followed by an evaluation of the model. Args: args: The object that contains all the parsed CLI arguments. """ print(colored("\nReading input data...", "green")) training_files = [] model = None if os.path.isdir(args.training_file_path): training_dir = args.training_file_path for f in os.listdir(training_dir): if f.endswith(".lsvm"): training_files.append(training_dir + f) for idx, train_file in enumerate(training_files): args.training_file_path = train_file input_dict = read_input_data("train", args) if idx == 0: if args.model_type == "et": model = ExtraTreesModel() elif (args.model_type == "mlp"): model = MlpModel() elif (args.model_type == "cnn"): model = CnnModel(input_dict) model.build_new_model() else: model.preprocess_input(input_dict) training_dict = model.train(input_dict) print_training_report(training_dict) else: input_dict = read_input_data("train", args) if args.model_type == "et": model = ExtraTreesModel() elif (args.model_type == "mlp"): model = MlpModel() elif (args.model_type == "cnn"): model = CnnModel(input_dict) elif (args.model_type == "rnn"): model = GruModel(input_dict) model.build_new_model() training_dict = model.train(input_dict) print_training_report(training_dict) testing_dict = model.test(input_dict) print_testing_report(testing_dict) model.save_model(args.output_model_path)
def predict(args): from c_py_interface.c_interface import CLibInterface if (args.model_type == "et"): model = ExtraTreesModel() elif (args.model_type == "mlp"): model = MlpModel() elif (args.model_type == "cnn"): model = CnnModel() model.load_model(args.model_path) clib = CLibInterface(args.c_library) clib.open_file(args.prediction_file_path) dict_input = {} dict_input['softmax'] = args.use_softmax_output dict_input['prediction'] = {} while (True): N = clib.read_next() if (N < 0): break for i in range(0, N): k = clib.count_roads(i) if (k > 0): roads = clib.read_roads(k, i) dict_input['prediction']['data'] = roads pred = model.predict(dict_input) preds = pred['predictions'] clib.write_roads(preds, i)
def test_model(args): """ Evaluates the model with the input data specified in the CLI arguments. Args: args: The object that contains all the parsed CLI arguments. """ print(colored("\nReading input data...", "green")) input_dict = read_input_data("test", args) model = None if (args.model_type == "et"): model = ExtraTreesModel() elif (args.model_type == "mlp"): model = MlpModel() elif (args.model_type == "cnn"): model = CnnModel(input_dict) elif (args.model_type == "rnn"): model = GruModel(input_dict) model.load_model(args.model_path) testing_dict = model.test(input_dict) print_testing_report(testing_dict)
def train_model(args): """ Trains the model with the input data specified in the CLI arguments followed by an evaluation of the model. Args: args: The object that contains all the parsed CLI arguments. """ print(colored("\nReading input data...", "green")) input_dict = read_input_data("train", args) model = None if args.model_type == "et": model = ExtraTreesModel() elif (args.model_type == "mlp"): model = MlpModel() elif (args.model_type == "cnn"): model = CnnModel(in_dict=input_dict) model.build_new_model() training_dict = model.train(input_dict) testing_dict = model.test(input_dict) print_training_report(training_dict) print_testing_report(testing_dict) model.save_model(args.output_model_path)
def parameter_search_GLOBAL(paddVal, numConvLayers, numFilters, kernelSize, numFcLayers, neurons, batch, epochs, dropRate, learnRate, classificType, aaSize, modelNum): for numConvL in numConvLayers: for numFcL in numFcLayers: for kernel in kernelSize: for learn in learnRate: # for classif in classificType: # para a camada 1 avalia 2 numfiltros diferentes, camada 2 avalia 4 combinações diferentes e para 3 avalia 8 combinações de filtros diferentes for i in range(0, 2**numConvL): filters = random.sample(numFilters, k=numConvL) for j in range(0, 2**numFcL): neur = random.sample(neurons, k=numFcL) for drop in dropRate: print('\nWith Dropout:') path = "models/savedModels/model_" + str( modelNum) + ".h5" modelName = str( modelNum ) + '_GLOBAL_Conv:' + str(filters) + '-' + str( kernel) + "x" + str(kernel) + "_FC:" + str( neur) + "_Drop:" + str( drop) + "_Learn:" + str(learn) modelNum += 1 print(modelName) model = CnnModel(paddVal, numConvL, filters, kernel, [], numFcL, neur, batch, epochs, drop, learn, 'FC', path, aaSize, 'global', 'yes', 'fc') save_results([modelName] + model.results)
def predict(args): if (args.model_type == "et"): model = ExtraTreesModel() elif (args.model_type == "mlp"): model = MlpModel() elif (args.model_type == "cnn"): model = CnnModel() model.load_model(args.model_path) # TODO continue implementation raise NotImplementedError
def parameter_search_GAP(paddVal, numConvLayers, numFilters, kernelSize, poolSize, batch, epochs, learnRate, classificType, aaSize, modelNum): for numConvL in numConvLayers: for kernel in kernelSize: for learn in learnRate: # for classif in classificType: # para a camada 1 avalia 2 numfiltros diferentes, camada 2 avalia 4 combinações diferentes e para 3 avalia 8 combinações de filtros diferentes for i in range(0, 2**numConvL): filters = random.sample(numFilters, k=numConvL) for pool in poolSize: path = "models/savedModels/model_" + str( modelNum) + ".h5" modelName = str(modelNum) + '_GAP' + "_Conv:" + str( filters) + '-' + str(kernel) + "x" + str( kernel) + '_Pool:' + str( pool) + "_Learn:" + str(learn) modelNum += 1 print(modelName) model = CnnModel(paddVal, numConvL, filters, kernel, pool, [], [], batch, epochs, [], learn, 'FC', path, aaSize, 'local', 'no', 'gap') save_results([modelName] + model.results)
def parameter_search_LOCAL(paddVal, activFun, numConvLayers, numFilters, kernelSize, poolSize, numFcLayers, neurons, batch, epochs, dropRate, learnRate, classificType, aaSize, modelNum, archNum): for numConvL in numConvLayers: for numFcL in numFcLayers: for kernel in kernelSize: for learn in learnRate: # for classif in classificType: # para a camada 1 avalia 2 numfiltros diferentes, camada 2 avalia 4 combinações diferentes e para 3 avalia 8 combinações de filtros diferentes for i in range(0, 2**numConvL): filters = random.sample(numFilters, k=numConvL) for j in range(0, 2**numFcL): neur = random.sample(neurons, k=numFcL) for pool in poolSize: for drop in dropRate: for l in range(1, 5, 3): # To test different datasets # l=1 - dataset is the negatives # l=2 - dataset is the benchmark (suppAB) # l=3 - dataset is the benchmark (suppCD) # l=4 - dataset is the benchmark (suppE) print('\nDATA: ' + str(l)) print('\nOPT:1') # Optimizer 1 - adam # Relu activation Function path = "models/savedModels/model_" + str( modelNum) + ".h5" modelName = str( modelNum ) + '_LOCAL' + "_Conv:" + str( filters ) + '-' + str(kernelSize) + "x" + str( kernelSize) + '_Pool:' + str( poolSize) + "_FC:" + str( neur) + "_ActivFun:" + str( activFun[0] ) + "_Drop:" + str( dropRate ) + "_Learn:" + str( learnRate ) + "_Opt:" + str( 1 ) + "_DataUsed:" + str(l) print(modelName) model = CnnModel( paddVal[l - 1], activFun[0], numConvL, filters, kernel, pool, numFcL, neur, batch, epochs, drop, learn, path, aaSize, archNum, 1, 1, l) save_results([modelName] + model.results) modelNum += 1 # Swish activation Function path = "models/savedModels/model_" + str( modelNum) + ".h5" modelName = str( modelNum ) + '_LOCAL' + "_Conv:" + str( filters ) + '-' + str(kernelSize) + "x" + str( kernelSize) + '_Pool:' + str( poolSize) + "_FC:" + str( neur) + "_ActivFun:" + str( activFun[1] ) + "_Drop:" + str( dropRate ) + "_Learn:" + str( learnRate ) + "_Opt:" + str( 1 ) + "_DataUsed:" + str(l) print(modelName) model = CnnModel( paddVal[l - 1], activFun[1], numConvL, filters, kernel, pool, numFcL, neur, batch, epochs, drop, learn, path, aaSize, archNum, 1, 1, l) save_results([modelName] + model.results) modelNum += 1 # Optimizer 3 - rmsProp # Relu activation Function print('\nOPT:3') path = "models/savedModels/model_" + str( modelNum) + ".h5" modelName = str( modelNum ) + '_LOCAL' + "_Conv:" + str( filters ) + '-' + str(kernelSize) + "x" + str( kernelSize) + '_Pool:' + str( poolSize) + "_FC:" + str( neur) + "_ActivFun:" + str( activFun[0] ) + "_Drop:" + str( dropRate ) + "_Learn:" + str( learnRate ) + "_Opt:" + str( 3 ) + "_DataUsed:" + str(l) print(modelName) model = CnnModel( paddVal[l - 1], activFun[0], numConvL, filters, kernel, pool, numFcL, neur, batch, epochs, drop, learn, path, aaSize, archNum, 1, 3, l) save_results([modelName] + model.results) modelNum += 1 # Swish activation Function path = "models/savedModels/model_" + str( modelNum) + ".h5" modelName = str( modelNum ) + '_LOCAL' + "_Conv:" + str( filters ) + '-' + str(kernelSize) + "x" + str( kernelSize) + '_Pool:' + str( poolSize) + "_FC:" + str( neur) + "_ActivFun:" + str( activFun[1] ) + "_Drop:" + str( dropRate ) + "_Learn:" + str( learnRate ) + "_Opt:" + str( 3 ) + "_DataUsed:" + str(l) print(modelName) model = CnnModel( paddVal[l - 1], activFun[1], numConvL, filters, kernel, pool, numFcL, neur, batch, epochs, drop, learn, path, aaSize, archNum, 1, 3, l) save_results([modelName] + model.results) modelNum += 1
# print(modelName) # model=CnnModel(paddVal[dataToLoad-1],actFun[0],len(filters),filters,kernelSize[0],poolSize[0],len(neur),neur,batch,epochs,dropRate[0],learnRate[0],path,dictionarySize,archNum,1,opt,dataToLoad,concatOrMulti) # save_results([modelName]+model.results) modelNum = modelNum + 1 path = "models/savedModels/model_" + str(modelNum) + ".h5" modelName = str(modelNum) + '_LOCAL' + "_Conv:" + str(filters) + '-' + str( kernelSize[0]) + "x" + str(kernelSize[0]) + '_Pool:' + str( poolSize[0]) + "_FC:" + str(neur) + "_ActivFun:" + str( actFun[1]) + "_Drop:" + str(dropRate) + "_Learn:" + str( learnRate) + "_Opt:" + str(3) + "_DataUsed:" + str( dataToLoad) + '_' + str(concatOrMulti) print(modelName) model = CnnModel(paddVal[dataToLoad - 1], actFun[0], len(filters), filters, kernelSize[0], poolSize[0], len(neur), neur, batch, epochs, dropRate[0], learnRate[0], path, dictionarySize, archNum, 1, 3, dataToLoad, concatOrMulti) save_results([modelName] + model.results) modelNum = modelNum + 1 path = "models/savedModels/model_" + str(modelNum) + ".h5" modelName = str(modelNum) + '_ConvLstm' + "_Conv:" + str( filters) + '-' + str(kernelSize[0]) + "x" + str( kernelSize[0]) + '_Pool:' + str( poolSize[0]) + '_LSTM:80' + "_ActivFun:" + str( actFun[0]) + "_Learn:" + str(learnRate[0]) + "_Opt:" + str( opt) + "_DataUsed:" + str(dataToLoad) + '_' + str( concatOrMulti) print(modelName) model = ConvLstm(paddVal[dataToLoad - 1], filters, kernelSize[0],