def main(train_models: bool, test_models: bool) -> None: signal.signal(signal.SIGINT, signal_handler) if train_models: print('---------------- TRAINING -----------------') for config in MODELS_CONFIG: try: train(config) except SIGINTError: print('\n\nTraining stopped. Skipping to the next model.\n\n') except Exception as e: print(e) traceback.print_exc() print('\n\nSkipping to the next model.\n\n') if test_models: test_output_dir = './test_results' print('---------------- TESTING ----------------') print('Warning! test_generation_200 disabled!') test_one_step = TestLotterOneStep(MODELS_CONFIG, test_output_dir) test_generation_20 = TestGeneration20(MODELS_CONFIG, test_output_dir) test_generation_200 = TestGeneration200(MODELS_CONFIG, test_output_dir) print('Test one-step-ahead') test_one_step.run() print('Test 20-step-ahead') test_generation_20.run() print('Test generation 200') test_generation_200.run()
def main(args): size = args.input_size base_dir = os.path.join('data/', str(size)) if size == 12: net = 'PNet' net_factory = net_factorys[0] end_epoch = FLAGS.end_epoch[0] elif size == 24: net = 'RNet' net_factory = net_factorys[1] end_epoch = FLAGS.end_epoch[1] elif size == 48: net = 'ONet' net_factory = net_factorys[2] end_epoch = FLAGS.end_epoch[2] model_path = os.path.join('model/', net) if not os.path.exists(model_path): os.mkdir(model_path) prefix = os.path.join(model_path, net) display = FLAGS.display lr = FLAGS.lr train(net_factory, prefix, end_epoch, base_dir, display, lr, restore_model=False) # restore_model=False is added by sherk
def main(parser): opt = parser.parse_args() with torch.autograd.profiler.profile(use_nvprof=opt['use_nvprof'], trace_path=opt['trace_path']) as prof: train(parser) print(prof.total_average()) sort_cpu = sorted(prof.key_averages(), key=lambda k: k.cpu_time) sort_cuda = sorted(prof.key_averages(), key=lambda k: k.cuda_time) def cpu(): for e in sort_cpu: print(e) def cuda(): for e in sort_cuda: print(e) cpu() if opt['debug']: print('`cpu()` prints out cpu-sorted list, ' '`cuda()` prints cuda-sorted list') pdb.set_trace()
def train(): extract_embeddings.extract_embeddings_func("dataset", "output/embeddings.pickle", "face_detection_model", "openface_nn4.small2.v1.t7") train_model.train("output/embeddings.pickle", "output/recognizer.pickle", "output/le.pickle") return "blah"
def process(): s_print('[process] Starting Process While Loop') while not exit_flag.wait(timeout=DETAILED_RECORD_PROCESSING_DELAY): if q.qsize() >= DETAILED_PROCESSING_BATCH_SIZE: count = DETAILED_PROCESSING_BATCH_SIZE data = list() while count > 0: d = pop_queue() data.append(d) q.task_done() count -= 1 train(data)
def main(): """ main """ parser = ConfigParser() parser.read('config_spss.ini') ''' # read train data [train_data,train_targets,train_clv] = read_data.training_data(parser) # read val data [val_data,val_targets,val_clv] = read_data.validation_data(parser) # read test data [test_data,test_targets,test_clv] = read_data.test_data(parser) ''' # read train data [train_data, train_targets, train_clv] = gen_randdata.gen_randdata(parser) # print(train_targets) # read val data [val_data, val_targets, val_clv] = gen_randdata.gen_randdata(parser) # read test data [test_data, test_targets, test_clv] = gen_randdata.gen_randdata(parser) train_data_list = [train_data, train_targets, train_clv] val_data_list = [val_data, val_targets, val_clv] test_data_list = [test_data, test_targets, test_clv] # Train DNN # Config Variables init_meth = parser['strs'].get('init_meth') olfn = parser['strs'].get('oplayer_fn') f = ['tanh', 'tanh', olfn] # Ints din = parser['ints'].getint('din') dh1 = parser['ints'].getint('dh1') dh2 = parser['ints'].getint('dh2') dh3 = parser['ints'].getint('dh3') dout = parser['ints'].getint('dout') # Instatiate the class l1 = srnlayer.SimpleRecurrentLayer(dh1, din, init_meth, parser) l2 = srnlayer.SimpleRecurrentLayer(dh2, dh1, init_meth, parser) l3 = feedforwardlayer.FeedForwardLayer(dout, dh2, init_meth) l = [l1, l2, l3] # Call train_mlp function train_model.train(train_data_list, val_data_list, test_data_list, l, f, parser) # Garbage collect gc.collect()
def Try(self, model_number): self.train_temp = [] with open("update_train.txt") as tdata: while True: line = tdata.readline() if not line: break self.train_temp.append([float(i) for i in line.split()]) self.train_temp = np.array(self.train_temp) self.result_temp = [] with open("update_result.txt") as rdata: while True: line = rdata.readline() if not line: break self.result_temp.append([float(i) for i in line.split()]) self.result_temp = np.array(self.result_temp) self.y = np.mat(self.result_temp) self.y = torch.tensor(self.y).float() run = train_model.train() run.update(model_number, self.train_temp, self.y) # go = try_update() # go.Try(1) # go.Try(2)
def train_new_model(action_array): print("_________________ CANNOT FIND A MATCH _______________") # return 0 # append_new_model(action_array) # print("+++++++++++++++++++ train(action_array) :",train(action_array)) # return random.uniform(0.8,1) return train(action_array)
def main(epoch, learn_rate, train_rate, batch_size, l2): # Download dataset and organize it pd.prep_data() # Pipeline trainset = tf.data.Dataset.list_files('data/train/*.png') trainset = trainset.shuffle(BUFFER_SIZE) trainset = trainset.map(ld.load_train) trainset = trainset.batch(1) testset = tf.data.Dataset.list_files('data/test/*.png') testset = testset.map(ld.oad_test) testset = testset.batch(1) # generator = GAN.Generator((256, 256, 3)) discriminator = GAN.Discriminator((256, 256, 3), (256, 256, 3)) gen_optimizer = tf.keras.optimizers.Adam(learning_rate=2e-4, beta_1=0.5, beta_2=0.99, epsilon=epsilon) dis_optimizer = tf.keras.optimizers.Adam(learning_rate=2e-4, beta_1=0.5, beta_2=0.99, epsilon=epsilon) # Checkpoint checkpoint_dir = 'data/training_checkpoints' checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt") checkpoint = tf.train.Checkpoint(generator_optimizer=gen_optimizer, discriminator_optimizer=dis_optimizer, generator=generator, discriminator=discriminator) # Train tm.train(trainset, EPOCHS) # Test # restoring the latest checkpoint in checkpoint_dir checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir)) # Run the trained model on the entire test dataset for inp, tar in testset.take(5): generate_images(generator, inp, tar)
def auth_process(): name = get_auth_from_face() #grab name if we can if name == "n/a": #no face in frame return "n/a" elif name == "unknown": #unknown face in frame to_speak = "want to make a profile?" mouth.speak_aloud(to_speak) if ears.listen_for_trigger(["yes", "yeah", "okay", 'sure'], time=3): to_speak = "What would you like me too call you?" mouth.speak_aloud(to_speak) new_name = ears.listen_and_translate(time=5) mouth.speak_aloud("Look into the Camera") face_data(new_name, 1) extract_embeddings() train() #train the model name = new_name else: to_speak = "Hello " + name + ", awaiting commands" mouth.speak_aloud(to_speak) mouth.speak_aloud(vc.decode_and_run(ears.listen_and_translate(5)))
def main(): # Parse input data args = parsing_inputs() # Obtain the dataloaders and a dictionary class_to_idx we will use during prediction trainloader, validloader, testloader, class_to_idx = load_data(args) # Now we download the model based on the input and select the device we will train it on possible_inputs = {'vgg16': 25088, 'alexnet': 9216} model, device = build_model(possible_inputs, args) # The next step is to define the criterion and train the model criterion = nn.NLLLoss() train(model, device, args, trainloader, validloader, criterion) # We then perform a validation test on new unseen data with torch.no_grad(): validation_test(model, testloader, device, criterion) # Finally we save the checkpoint save_check(args, model, class_to_idx, possible_inputs)
def main(args): prediction = args.prediction train = args.train if prediction: test_path = args.data_dir test_file = args.test_file context = args.Context res_dir = args.res_dir test_SNPs = pd.read_csv(test_path + '/' + test_file, sep='\t') test_SNPs = test_SNPs[['Chromosome','Start','End']] data = score.get_score(test_SNPs) ''' kde get transformed score ''' trans_score = transform.transform(data, context) ''' load WEVar model, and predict WEVar score ''' res = model.WEVar(trans_score, context) test_SNPs['WEVar_{}'.format(context)] = res test_SNPs.to_csv('{}/{}'.format(res_dir,test_file), index=False, sep='\t') elif train: data_path = args.data_dir train_file = args.train_file train_SNPs = pd.read_csv(data_path + '/' + train_file, sep='\t') X = train_SNPs[['Chromosome', 'Start', 'End']] X = score.get_score(X) Y = train_SNPs['Labels'].to_numpy() train_model.train(X,Y, train_file) train_model.test(X,Y, train_file)
def main(parser): opt = parser.parse_args() if opt['torch']: with torch.autograd.profiler.profile() as prof: train(parser) print(prof.total_average()) sort_cpu = sorted(prof.key_averages(), key=lambda k: k.cpu_time) sort_cuda = sorted(prof.key_averages(), key=lambda k: k.cuda_time) def cpu(): for e in sort_cpu: print(e) def cuda(): for e in sort_cuda: print(e) cpu() if opt['debug']: print('`cpu()` prints out cpu-sorted list, ' '`cuda()` prints cuda-sorted list') pdb.set_trace() else: pr = cProfile.Profile() pr.enable() train(parser) pr.disable() s = io.StringIO() sortby = 'cumulative' ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() print(s.getvalue()) if opt['debug']: pdb.set_trace()
def executeButtonFunction(self): if "" == self.data_path.toPlainText(): print("Set Data Path") return if config["mode"] == "collect_image": config["driver_path"] = os.getcwd() + r'\chromedriver_win32\chromedriver.exe' config["image_save_path"] = self.data_path.toPlainText() print(config["image_save_path"]) config["keyword"] = self.keyword.text() download_google_staticimages(config) elif config["mode"] == "train_model": config["image_folder"] = self.data_path.toPlainText() config["epoch"] = int(self.epoch.text()) config["learning_rate"] = float(self.learning_rate.text()) config["weight_decay"] = float(self.weight_decay.text()) train() elif config["mode"] == "shopping_lense": config["test_image"] = self.data_path.toPlainText() test() self.showResult()
def on_new_client(conn, addr): data = conn.recv(1024) data = data.decode("utf-8").strip() file = data global data2 data2 = "students/" + data + ".mp4" print(data) print(data2) with open(data2, 'wb') as f: print('file opened') while True: data = conn.recv(1024) print('data=%s', (data)) if not data: break else: f.write(data) print('Done sending') im.makeimage(file) tr.train(file) print('Got connection from', addr) f.close()
def main(args): size = args.input_size base_dir = os.path.join('g:/mtcnn-dataset/data/', str(size)) if size == 12: net = 'PNet' net_factory = net_factorys[0] end_epoch = FLAGS.end_epoch[0] elif size == 24: net = 'RNet' net_factory = net_factorys[1] end_epoch = FLAGS.end_epoch[1] elif size == 48: net = 'ONet' net_factory = net_factorys[2] end_epoch = FLAGS.end_epoch[2] model_path = os.path.join('../model/', net) if not os.path.exists(model_path): os.mkdir(model_path) prefix = os.path.join(model_path, net) display = FLAGS.display lr = FLAGS.lr train(net_factory, prefix, end_epoch, base_dir, display, lr)
def start_train(model, train_iter, dev_iter, test_iter): print("\n cpu_count \n", mu.cpu_count()) torch.set_num_threads(config.num_threads) if os.path.exists("./Test_Result.txt"): os.remove("./Test_Result.txt") model_training, score = train_model.train(train_iter, dev_iter, test_iter, model, config) resultlist = [] if os.path.exists("./Test_Result.txt"): file = open("./Test_Result.txt") for line in file.readlines(): if line[:10] == "Evaluation": resultlist.append(float(line[34:41])) result = sorted(resultlist) file.close() file = open("./Test_Result.txt", "a") file.write("\nThe Best Result is : " + str(result[len(result) - 1])) file.write("\n") file.close()
pt.pytesseract.tesseract_cmd = 'C:/Program Files (x86)/Tesseract-OCR/tesseract' print('started!!!') ##create the game object game = CHAIRFED() print("game object created") epoch = 2 # Number of games played in training, ##switch to toggle between train and test. train_mode = 0 if train_mode == 1: # Train the model hist,loss = train(game, model, epoch, verbose=1) print(loss) np.savetxt('loss_history.txt', loss) print("Training done") else: # Test the model hist = test(game, model, epoch, verbose=1) print("Testing done") print('finished!!!') print(hist) np.savetxt('win_history.txt', hist) plt.plot(moving_average_diff(hist)) plt.ylabel('Average number of stableness per quater') plt.show()
# path for save the data model_root_dir = "/zfssz6/ST_MCHRI/BIGDATA/P18Z10200N0124/NIPT_CNV/NIPT_CNV/npz_feature_file/winsize_1000" total_samples_ls_fn = "/zfssz6/ST_MCHRI/BIGDATA/P18Z10200N0124/NIPT_CNV/sample.list" min_r = 0.1 min_f_deldup = 0.01 min_f_neu = -2 is_norm = False win_size = 1000 batch_size=256 test_ratio = 0.1 validation_ratio=0.1 example_min_size = 160000 # x_train, y_train = load_train_data(model_root_dir, total_samples_ls_fn, # win_size, min_r, min_f_deldup, min_f_neu, is_norm, test_ratio, example_min_size) # X, y=my_generator(model_root_dir, total_samples_ls_fn, # win_size, min_r, min_f_deldup, min_f_neu, is_norm, test_ratio, example_min_size) train_generator,validation_generator = my_generator(model_root_dir, total_samples_ls_fn, win_size, min_r, min_f_deldup, min_f_neu, is_norm, test_ratio=test_ratio, validation_ratio=validation_ratio, batch_size=batch_size, n_classes = 3, shuffle='Local_shuffle') # print(len(X)) # multi gpu train train(train_generator,validation_generator, model_root_dir) # cv_train(x_train, y_train, model_root_dir) # mnd_train(x_train, y_train, model_root_dir)
background_test_path = args.test_path # 读取参考训练数据和测试数据 print("[INFO] loading background images...") x_train_background, y_train_background = load_images( background_train_path, IMAGE_SIZE) x_test_background, y_test_background = load_images( background_train_path, IMAGE_SIZE) # 初始化模型 print("[INFO] initialize background model...") background_model = creat_model(FILTERS, KERNEL_SIZE, INPUT_SHAPE, POOL_SIZE, len(os.listdir(background_train_path))) # 训练模型 print("[INFO] compiling background model...") train(background_model, x_train_background, y_train_background, x_test_background, y_test_background, len(y_train_background) // 10, EPOCHS, 'models/backgroud_model.h5') elif args.function == 2: # 使用迁移学习训练比赛集 # 比赛训练数据集顶层类型(大类) evaluation_top_train_path = args.train_path # 比赛测试数据集 evaluation_top_test_path = args.test_path # 读取比赛训练数据和测试数据 print("[INFO] loading evaluation images...") x_train_top_evaluation, y_train_top_evaluation = load_images( evaluation_top_train_path, IMAGE_SIZE) x_test_top_evaluation, y_test_top_evaluation = load_images( evaluation_top_test_path, IMAGE_SIZE) # 加载并设置模型 print('[INFO] loading model...')
import os, shutil from keras import backend import settings from train_model import train from evaluate_model import evaluate for config in settings.TESTS_CONFIG: print("Training model ", str(config)) train(config) file = os.path.basename(config.file_name) os.rename(config.file_name, os.path.join(settings.completed_tests_folder, file)) shutil.move(config.model_dir, os.path.join(settings.results_folder, str(config))) print("Evaluating model ", str(config)) evaluate(config) backend.clear_session()
def main(): prior = torch.tensor(opt.prior) input_size = 32 # prepare training data transform = { 'train': transforms.Compose([ transforms.Resize(input_size), transforms.RandomCrop(input_size, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]), 'val': transforms.Compose([ transforms.Resize(input_size), transforms.CenterCrop(input_size), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]), } label_trainset = datasets.CIFAR10(root=opt.label_dir, train=True, download=True, transform=transform['train']) label_testset = datasets.CIFAR10(root=opt.label_dir, train=False, download=True, transform=transform['val']) labelset_index = [] classnum = np.ones(opt.num_classes) * opt.pos_num i = 0 while (np.sum(classnum) > 0): image = label_trainset[i][0] label = label_trainset[i][1] if (classnum[label] > 0): labelset_index.append(i) classnum[label] -= 1 i += 1 label_train_subset = torch.utils.data.Subset(label_trainset, labelset_index) target_transform = transforms.Lambda( lambda target: target + opt.num_classes) unlabel_set = datasets.ImageFolder(opt.unlabel_dir, transform['train'], target_transform=target_transform) trainloader = torch.utils.data.DataLoader(label_train_subset + unlabel_set, batch_size=opt.pu_batchsize, shuffle=True, num_workers=32) testloader = torch.utils.data.DataLoader(label_testset, batch_size=opt.pu_batchsize, shuffle=False, num_workers=32) dataloader = {} dataloader['train'] = trainloader dataloader['val'] = testloader # stage1: get positive data from unlabeled dataset model_pu = nn.DataParallel( initialize_model(use_pu=True, num_classes=opt.num_classes)).cuda() optimizer_pu = optim.SGD(model_pu.parameters(), lr=opt.pu_lr, momentum=opt.momentum, weight_decay=opt.pu_weight_decay) scheduler_pu = optim.lr_scheduler.MultiStepLR(optimizer_pu, milestones=[50, 100, 150], gamma=0.1, last_epoch=-1) model_pu = train_pu(model_pu, dataloader, optimizer_pu, scheduler_pu, prior=prior, num_classes=opt.num_classes, num_epochs=opt.pu_num_epochs) trainloader2 = torch.utils.data.DataLoader(unlabel_set, batch_size=opt.pu_batchsize, shuffle=False, num_workers=32) positive_index_all = get_positive(model_pu, trainloader2) print("We have {} positive unlabeled images for all!".format( len(positive_index_all))) unlabel_positive_set = torch.utils.data.Subset(unlabel_set, positive_index_all) trainloader3 = torch.utils.data.DataLoader(label_train_subset + unlabel_positive_set, batch_size=opt.batchsize, shuffle=True, num_workers=32) dataloader['train'] = trainloader3 # stage2: train student model with rkd method teacher = nn.DataParallel( initialize_model(use_pu=False, num_classes=opt.num_classes)).cuda() student = nn.DataParallel(ResNet18(num_classes=opt.num_classes)).cuda() class_weight = get_class_weight(teacher, trainloader3, num_classes=opt.num_classes) print(class_weight) class_weights = perturb(class_weight, opt.epsilon, opt.perturb_num) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(student.parameters(), lr=opt.lr, momentum=opt.momentum, weight_decay=opt.weight_decay) scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[50, 100, 150], gamma=0.1, last_epoch=-1) model_s, hist = train(student, teacher, class_weights, dataloader, criterion, optimizer, scheduler, num_epochs=opt.num_epochs)
def Predict(): pre = train_model.train() pre.predict()
def NewModel(): New = train_model.train() var3 = New.train_model(3) var1 = New.train_model(1) var2 = New.train_model(2) return var1, var2, var3
def main(): #mypath = r'/home/connlab/108IR/will/final/NVSM_pytorch/' mypath = r'C:/Users/willll/Desktop/WIillll/IRCLass/Final/NVSM_pytorch' pretrained_model = 'bert-base-uncased' glove_path = Path(mypath + '/glove') model_folder = Path(mypath + '/models') # data_folder = Path(mypath + '/data/processed') data_folder = Path(mypath + '/Willll/fakedoc') testing_query_folder = Path(mypath + '/Willll/test/query') model_path = model_folder / 'nvsm_bert.pt' batch_size = 140 # for 150, 8053 / 8113MB GPU memory, to tweak epochs = 1 docs, queries, tokenizer = load_data(data_folder, testing_query_folder, pretrained_model) # docs = docs[:20] doc_names = [doc['name'] for doc in docs] n_grams, document_ids = create_dataset( tok_docs=[doc['tokens'] for doc in docs], tokenizer=tokenizer, n=30) print('N-grams number', len(n_grams)) k_values = [1, 3, 5, 10] (train_data, eval_data, eval_train_data) = create_pytorch_datasets(n_grams, document_ids) print('Train dataset size', len(train_data)) print('Eval dataset size', len(eval_data)) print('Eval (training) dataset size', len(eval_train_data)) train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True) eval_loader = DataLoader(eval_data, batch_size=batch_size, shuffle=False) eval_train_loader = DataLoader(eval_train_data, batch_size=batch_size, shuffle=False) device = torch.device('cuda') lamb = 1e-3 nvsm = NVSMBERT( pretrained_model=pretrained_model, n_doc=len(doc_names), dim_doc_emb=20, neg_sampling_rate=10, ).to(device) # BERT custom optimizer param_optimizer = list(nvsm.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] optimizer = BertAdam(params=optimizer_grouped_parameters, lr=5e-5, warmup=0.1, t_total=len(train_loader) * epochs) train(nvsm=nvsm, device=device, optimizer=optimizer, epochs=epochs, train_loader=train_loader, eval_loader=eval_train_loader, k_values=k_values, loss_function=loss_function, lamb=lamb, print_every=10000) torch.save(nvsm.state_dict(), model_path) nvsm.eval() recall_at_ks = evaluate( nvsm=nvsm, device=device, eval_loader=eval_loader, recalls=k_values, loss_function=loss_function, ) print(generate_eval(k_values, recall_at_ks)) queries_text = [query['tokens'] for query in queries] queries_name = [query['name'] for query in queries] evaluation_results = evaluate_queries_bert(nvsm, queries_text, doc_names, tokenizer, batch_size, device) print(evaluation_results) # print(len(ranksResults)) for query_name, query_text, doc_idx in zip(queries_name, queries_text, evaluation_results): print(f'{query_name} {query_text:35} -> {doc_names[doc_idx]}') with open(mypath + './Willll/result.txt', 'w') as f: f.write('Query,RetrievedDocuments\n') resuList = ' ' for qIndex, qName in enumerate(queries_name): f.write(f'{qName},') f.write( f'{resuList.join(doc_names[x] for x in ranksResults[qIndex])}\n' )
btn2 = Button(root, text="체험자 사진 촬영", width=20, height=1, font=font, foreground='white', background=btncolor, command=lambda: shot(inputText.get())) btn3 = Button(root, text="인공지능 학습", width=20, height=1, font=font, foreground='white', background=btncolor, command=lambda: train(inputText.get())) btn4 = Button(root, text="스마트 도어 작동", width=20, height=1, font=font, foreground='white', background=btncolor, command=lambda: lock(inputText.get())) btn5 = Button(root, text="체험자 사진 삭제", width=20, height=1, font=font, foreground='white', background=btncolor,
def on_Button_train_clicked(self): print("Button train clicked") attributes_train = self.get_train_attributes() submit_status = True k_value = 0 if attributes_train[2] == "KNN": # pop up screen try: import pop_up_entry except: print("import error") pop_up1 = pop_up_entry.pop_up_entry(self.app, attributes_train[2]) k_value, submit_status = pop_up1.get_status() print(k_value) print(submit_status) if submit_status == True: #make stus bar busy self.update_status_bar() self.app.processEvents() # train model try: import train_model except: print("import exception") model1 = train_model.train(attributes_train, k_value, self.app, self.pca_model) self.points = model1.get_map_data_points() self.current_model = model1.get_model() #update screen self.accuracy_display.display( model1.get_model_accuracy()) # set the lcd accuract digit self.update_screen_widgets(attributes_train) #make model ready message self.app.processEvents() msg = QMessageBox() msg.setText('Model Ready') msg.exec_() try: import map_load_time except: print("import exception") numsamples = int(attributes_train[0]) loadtime1 = map_load_time.map_load_time() timetoloadmap = loadtime1.calculate_load_time(numsamples) # make map ready after a few seconds depending on sample size self.app.processEvents() time.sleep(int(timetoloadmap)) self.status_display_map.setText("READY") self.status_display_map.setStyleSheet( "QLabel { color: green ; font-weight: bold}") self.status_display_map.update() # make map ready message self.app.processEvents() msg = QMessageBox() msg.setText('Geo Map Ready') msg.exec_()
import train_model from model_vgg16 import Vggmodel import tensorflow as tf import time vgg = Vggmodel(optimizer=tf.train.GradientDescentOptimizer, fine_tuning=True, lr=0.001, dropout=True, adaptive_ratio=1.0) train_model.train(vgg, timestamp=int(time.time()))
if args.cpu: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "-1" else: # this will prevent TF from allocating the whole GPU from keras import backend as K import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) import data_generators import train_model import validate_model os.makedirs("data\\training", exist_ok=True) os.makedirs("data\\validation", exist_ok=True) os.makedirs("data\\results", exist_ok=True) if args.action[0] == "data_train": data_generators.generate_examples(args.original_path[0], "data\\training", args.count[0], True) elif args.action[0] == "data_val": data_generators.generate_examples(args.original_path[0], "data\\validation", args.count[0], False) elif args.action[0] == "train": train_model.train("data\\training") elif args.action[0] == "eval": validate_model.evaluate(args.model_name[0])
def classify(classifiers, strategies, report, deploy, results_dir, analysis_dir): """Executes the classification process. The classification process is defined by a sequence of steps: Start ➜ Gather Data ➜ Prepare Data ➜ Train Models ➜ Deploy Models and Performances ➜ End Args: classifiers: A list of classifiers to train. strategies: A list of strategies to use during training. report: Boolean value used to define if models peformances will be reported. deploy: Boolean value used to define if models trained objects will be deployed. results_dir: A string path to directory where dataframes, models and performances will be saved. analysis_dir: A string path to directory where raw spreadsheets are available. """ ########################## # START # ########################## text_column = 'Paragraph' classes = ['CF – Contribution flow', 'CT – Choose a task', 'FM – Find a mentor', 'TC – Talk to the community', 'BW – Build local workspace', 'DC – Deal with the code', 'SC – Submit the changes'] classifiers_available = { 'rf': RandomForestClassifier(), 'svc': LinearSVC(), 'mnb': MultinomialNB(), 'knn': KNeighborsClassifier(), 'lr': LogisticRegression(), 'mlp': MLPClassifier(), 'sgd': SGDClassifier(), } strategies_available = { 'cc': 'classifier-chains', 'lp': 'label-powerset', 'ovr': 'one-vs-the-rest' } selected_classifiers = [classifiers_available[classifier] for classifier in classifiers] selected_strategies = [strategies_available[strategy] for strategy in strategies] ########################## # GATHER DATA # ########################## print("Gathering dataframe for classification.") dataframe = check_if_dataframe_copy_exists(results_dir, analysis_dir) ########################## # PREPARE DATA # ########################## print("Shuffling and splitting dataframe into training and test sets.") X_train, X_test, y_train, y_test = shuffle_and_split( dataframe, [text_column], classes) print("Converting paragraphs to statistical features.") X_train, X_test = vectorize_paragraphs( X_train[text_column].tolist(), X_test[text_column].tolist()) ################################### # TRAIN, REPORT AND DEPLOY MODELS # ################################### for classifier in selected_classifiers: classifier_name = type(classifier).__name__ training_args = { 'classifier': classifier, 'X_train': X_train, 'X_test': X_test, 'y_train': y_train, 'y_test': y_test, } for strategy in selected_strategies: print("Executing " + strategy + " classification.") print("Classifier: " + classifier_name) deployment_args = { 'strategy': strategy, 'classifier_name': classifier_name, 'results_dir': results_dir, } model, performance = train(strategy=strategy, **training_args) if report: report_performance(report=performance, **deployment_args) if deploy: deploy_model(model=model, **deployment_args)
save_dir = './model_saver/' isExists = os.path.exists(save_dir) if not isExists: os.makedirs(save_dir) save_path = save_dir + 'model' method = sys.argv[1] print("method", method) if method == "train": for r in range(10): print('#############################') print("Epoch", r, time.asctime(time.localtime(time.time()))) print('#############################') t1 = time.time() model_lstm = train(model_lstm, data_path, save_path, r, road) t2 = time.time() if r >= 0: print("testset acc: ", end='') count_acc(model_lstm, road, test_path) t3 = time.time() print("train cost time", t2 - t1, "acc cost time", t3 - t2) elif method == "test": epoch = sys.argv[2] saver = tf.train.Saver() saver.restore(model_lstm.sess, './model_saver/model-' + epoch) print("Epoch", epoch, "testset acc: ", end='') count_acc(model_lstm, road, test_path) elif method == "beam": epoch = sys.argv[2] K = int(sys.argv[3])