def setUpClass(cls): compiler = Compiler("test/compile.sh") cls.tmp_dirs = {} for test_c, test_fn in c_files_list: d = tempfile.TemporaryDirectory() file_test = os.path.join("test", test_c) file_actual = os.path.join(d.name, "actual.c") file_base = os.path.join(d.name, "base.c") file_target = os.path.join(d.name, "target.o") actual_preprocessed = preprocess(file_test, cpp_args=["-DACTUAL"]) base_preprocessed = preprocess(file_test, cpp_args=["-UACTUAL"]) strip_other_fns_and_write(actual_preprocessed, test_fn, file_actual) strip_other_fns_and_write(base_preprocessed, test_fn, file_base) actual_source = Path(file_actual).read_text() target_o = compiler.compile(actual_source, show_errors=True) assert target_o is not None shutil.copy2(target_o, file_target) os.remove(target_o) shutil.copy2("test/compile.sh", d.name) cls.tmp_dirs[(test_c, test_fn)] = d
def main(): print("PREPROCESSING DATA") preprocess() print("LOADING TOKENIZER") tokenizer = get_tokenizer() data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False) print("LOADING MODEL", cfg('model')) model = get_model(tokenizer) print("LOADING DATA") if cfg('encoding') == 'LBL': train_dataset = LBLDataset(tokenizer=tokenizer, file_path=filename('train')) elif cfg('encoding') == 'blocked': train_dataset = BlockedDataset(tokenizer=tokenizer, file_path=filename('train')) elif cfg('encoding') == 'text': train_dataset = TextDataset(tokenizer=tokenizer, file_path=filename('train'), block_size=cfg('max_block')) elif cfg('encoding').startswith('inter'): if cfg('encoding').endswith('LBL'): loader = LBLDataset elif cfg('encoding').endswith('blocked'): loader = BlockedDataset d1 = loader(tokenizer=tokenizer, file_path=filename('train')) d2 = loader(tokenizer=tokenizer, file_path=filename('dirty')) train_dataset = CombinedDataset(d1, d2) else: raise ValueError("Unkown encoding") trainer = get_trainer(train_dataset, data_collator, model) def validator(x, y): global BEST_metric model.save_pretrained(session) metric, pred = validate(model, tokenizer, x, y) if np.mean(metric) > BEST_metric: print("NEW BEST (saving)") BEST_metric = np.mean(metric) # save predicitions and model save(session + "metric.txt", str(metric) + "\n") save(session + "pred.txt", str(pred) + "\n\n") return metric, pred trainer.validator = validator trainer.val_dataset = get_validation_data() # saving configuration print("SAVING...") session = get_session_path() print(session) save(session + "conf.txt", repr(cfg())) print("STARTING TRAINING...") trainer.train()
def prepare_data(self) -> None: train_turns = preprocess(self.hparams.train_path, self.hparams.ontology_path) val_turns = preprocess(self.hparams.val_path, self.hparams.ontology_path) test_turns = preprocess(self.hparams.test_path, self.hparams.ontology_path) self.train_dataset = MultiWozDSTDataset(train_turns, self.tokenizer) self.val_dataset = MultiWozDSTDataset(val_turns, self.tokenizer) self.test_dataset = MultiWozDSTDataset(test_turns, self.tokenizer)
def __init__(self, hparams: Namespace): super().__init__() self.hparams = hparams self.criterion = nn.CTCLoss() if Path(self.hparams.tokenizer_path).exists(): self.tokenizer = WordLevelTokenizer(self.hparams.tokenizer_path) else: train_turns = preprocess(self.hparams.train_path, self.hparams.ontology_path) self.tokenizer = get_tokenizer(train_turns, self.hparams.tokenizer_path) # embedding self.embedding = nn.Embedding(self.tokenizer.get_vocab_size(), self.hparams.embedding_dim) self.pos_embedding = PositionalEncoding( d_model=self.hparams.embedding_dim, dropout=self.hparams.dropout) # value decoder self.value_decoder = nn.ModuleList([ nn.MultiheadAttention(embed_dim=self.hparams.hidden_dim, num_heads=self.hparams.num_heads, dropout=self.hparams.dropout) for _ in range(3) ]) self.vocab_proj = nn.Linear(self.hparams.hidden_dim, self.tokenizer.get_vocab_size())
def test_preprocess(): ''' Test utils to preprocess sentinel2-cube''' from src.preprocess import preprocess import numpy as np from xarray.core.dataset import Dataset import pandas as pd t = 5 B03 = np.random.rand(t, 100, 100) B08 = np.random.rand(t, 100, 100) CLP = np.zeros((t, 100, 100)) lon = np.repeat(-99.83, 100).tolist() lat = np.repeat(42.25, 100).tolist() cube = Dataset( { 'B03': (["time", 'lat', 'lon'], B03), 'B08': (["time", 'lat', 'lon'], B08), 'CLP': (["time", 'lat', 'lon'], CLP) }, coords={ 'lon': lon, 'lat': lat, 'time': pd.date_range('2014-09-06', periods=t), 'reference_time': pd.Timestamp('2014-09-05') }) cube, background_ndwi = preprocess(cube, max_cloud_proba=0.1, nans_how='any', verbose=1, plot_NDWI=False) assert hasattr(cube, "NDWI") assert background_ndwi.name == "NDWI" assert background_ndwi.data.shape == (100, 100)
def train(args): """ Train the final model Args: args (TYPE): Description Deleted Parameters: model_type (TYPE): Description """ logger.info('train final model, min_count=%s, extract_method=%s', args.min_count, args.feature_type) # preprocess all data preprocess.preprocess(CORPUS_FILES, skip_viTokenizer=False, export_pos=False, test_size=0., is_final=True) # build word2vec model w2v.train_model(min_count=args.min_count, use_external_data=False, is_final=True) # build tfidf model tfidf.train_tfidf(min_count=args.min_count, is_final=True) X_train, y_train, _, _ = compute_train_test_matrix(args.min_count, args.feature_type, is_final=True) params = { 'criterion': 'gini', 'min_samples_split': 5, 'n_estimators': 200, 'min_samples_leaf': 1, 'max_depth': 110, 'max_features': 'auto', 'bootstrap': False } clf = RandomForestClassifier(**params, random_state=42, verbose=2, n_jobs=-1) clf.fit(X_train, y_train) joblib.dump(clf, helper.get_model_path('RandomForest', is_final=True))
def test(data_dir): preprocess.preprocess(data_dir, False) model = '../bin/models/final_model.h5' test_generator = datagen.DataGenerator(constants.TEST_PROCESSED_DIR, constants.SEQUENCE_SIZE, constants.BATCH_SIZE, constants.CONTEXT_WINDOW_SIZE, 'x_[0-9]+.npy', 'y_[0-9]+.npy') note_f1s = [] note_precisions = [] note_recalls = [] frame_f1s = [] frame_precisions = [] frame_recalls = [] for sample in glob.glob(os.path.join(constants.TEST_PROCESSED_DIR, 'x_*')): isolated_filename = sample.split('/')[-1] gt_file= os.path.join(constants.TEST_PROCESSED_DIR,test_generator.corresponding_y(isolated_filename)) referances = np.load(gt_file, mmap_mode='r') predictions = infer_from_processed(model, sample) note_precision, note_recall, note_f1 = get_note_evaluation(referances, predictions) frame_precision, frame_recall, frame_f1 = get_f1_score_frames(referances, predictions) note_f1s.append(note_f1) note_precisions.append(note_precision) note_recalls.append(note_recall) frame_f1s.append(frame_f1) frame_precisions.append(frame_precision) frame_recalls.append(frame_recall) avg_frame_f1 = np.mean(frame_f1s) avg_frame_precision = np.mean(frame_precisions) avg_frame_recall = np.mean(frame_recalls) avg_note_f1 = np.mean(note_f1s) avg_note_precision = np.mean(note_precisions) avg_note_recall = np.mean(note_recalls) print('Frame:') print([avg_frame_precision, avg_frame_recall, avg_frame_f1]) print('Note:') print([avg_note_precision, avg_note_recall, avg_note_f1])
def main(bool_dict): """ Launch all project steps. :param bool_dict: Dictionnary with step names as keys and boolean as values allowing to bypass steps. This can be useful to re-run all steps but model training steps if they are already done for example. :return: """ download_file_from_url( files.GDP_ENERGY_DATA_URL, os.path.join(files.RAW_DATA, files.GDP_ENERGY_DATA_CSV)) if bool_dict["preprocess"]: preprocess() if bool_dict["lin_reg_train"]: lin_reg_train() if bool_dict["evaluate"]: evaluate()
def sentiment_classification(): """Run multi-label boardgame classification given boardgame description. --- parameters: - name: body in: body schema: id: description required: - description properties: description: type: [string] description: the required boardgame description for POST method required: true definitions: SentimentResponse: Project: properties: status: type: string ml-result: type: object responses: 40x: description: Client error 200: description: Multi-label Boardgame Categorization examples: [ { "status": "success", "sentiment": "1" }, { "status": "error", "message": "Exception caught" }, ] """ json_request = request.get_json() # Getting whatever you sent to service if not json_request: return Response("No json provided.", status=400) # If you sent nothing, we throw error description = json_request['description'] if description is None: return Response("No text provided.", status=400) else: preprocessed_description = preprocess(description) predicted_categories = predict(preprocessed_description) return flask.jsonify({"status": "success", "predicted_categories": predicted_categories.tolist()}) # Returning an answer to the POST request; the .jsonify part will put HTTP status 200
def test(opt): testdata, testGenerator, classes = preprocess(path='./data' + os.sep + opt.test, batchsize=opt.batchsize, imagesize=opt.imagesize, shuffle=False) images = [ i[0].split('/')[-1] for i in testGenerator.sampler.data_source.imgs ] device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model = Vgg(num_channels=3, num_classes=200, depth=opt.model, conv1_1=False, initialize_weights=True).to(device) print(iter(testGenerator).__next__()[0].size()) if opt.conv1_1 and opt.model: model = torch.load('./models' + os.sep + 'VdcnnIR_C11_' + str(opt.model)) else: model = torch.load('./models' + os.sep + 'VdcnnIR_' + str(opt.model)) model.eval() total_testpredictions = [] for idx_t, data_t in enumerate(testGenerator): data_t, _ = data_t[0], data_t[1] data_t = data_t.to(device) with torch.no_grad(): prob_t = model(data_t) pred_t = np.argmax(prob_t.detach().cpu(), -1) total_testpredictions.extend(pred_t.tolist()) print('Iter: [{}/{}]'.format(idx_t + 1, len(testGenerator))) if opt.conv1_1 and opt.model: with open('./results/model_{}_C11_Testpred'.format(opt.model), 'wb') as f: pickle.dump(total_testpredictions, f) else: with open('./results/model_{}_Testpred'.format(opt.model), 'wb') as f: pickle.dump(total_testpredictions, f) return images, total_testpredictions
def main(): # omoshiro_tweets = load_omoshiro_tweets_from_json(tweet_filename) valid_dataset_size = 128 batch_size = 8 config_path = 'cl-tohoku/bert-base-japanese-whole-word-masking' if torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') tokenizer = BertJapaneseTokenizer.from_pretrained(config_path) config = BertConfig.from_pretrained(config_path) pad_token_id = config.pad_token_id dataset = load_dataset(dataset_filename) train_dataset = dataset[:-128] valid_dataset = dataset[-128:] train_dataset, train_max_length = preprocess(train_dataset, tokenizer, config, batch_size=batch_size, device=device) train_dataset = normalize_dataset(train_dataset) valid_dataset, valid_max_length = preprocess(valid_dataset, tokenizer, config, batch_size=batch_size, device=device) valid_batches = mk_batches(dataset=valid_dataset, max_length=valid_max_length, batch_size=batch_size, device=device, pad=pad_token_id) print('Train dataset size is {}, Valid dataset size is {}'.format(len(train_dataset), len(valid_dataset))) model = BertPredictor(config_path=config_path, model_path=config_path) # model = Perceptron(vocab_size=tokenizer.vocab_size, hidden_size=128, device=device) model.to(device) criterion = torch.nn.CrossEntropyLoss(ignore_index=NEUTRAL) optimizer = optim.SGD(model.parameters(), lr=0.0001) for epoch in range(10): print('------ Epoch {} ------'.format(epoch + 1)) train_batches = mk_batches(dataset=train_dataset, max_length=train_max_length, batch_size=batch_size, device=device, pad=pad_token_id) print('Train') model.train() accuracy = 0.0 for i, batch in enumerate(train_batches): model.zero_grad() src = batch['src'] tgt = batch['tgt'] # output = [batch_size, vocab_size] output = model(src) loss = criterion(output, tgt) labels = torch.argmax(output, dim=-1) accuracy = ((labels == tgt).sum() + accuracy * i * batch_size) / ((i + 1) * batch_size) loss.backward() optimizer.step() sys.stdout.write('\rLoss: {}, Accuracy: {}'.format(loss.item(), accuracy)) # accuracy /= len(train_dataset) print('\nTrain accuracy {}'.format(accuracy)) print('Validation') model.eval() with torch.no_grad(): accuracy = 0.0 for batch in valid_batches: src = batch['src'] tgt = batch['tgt'] output = model(src) labels = torch.argmax(output, dim=-1) accuracy += (labels == tgt).sum() accuracy /= valid_dataset_size print('Valid accuracy : {}'.format(accuracy)) accuracy = 0.0 for batch in valid_batches: accuracy += (JUN == batch['tgt']).sum() accuracy /= valid_dataset_size print('== JUN accuracy : {}'.format(accuracy))
def predict(self, model_input): model_input = preprocess(model_input) predicted_target = self.knn_clf.predict(model_input) predicted_target = postprocess(predicted_target) return predicted_target
def fit(self, train_x, train_y): train_x = preprocess(train_x) self.knn_clf.fit(train_x, train_y)
type=str, default='test', choices=['preprocess', 'train', 'test', 'predict', 'select']) parser.add_argument('--gpu', type=int, default=0, choices=[i for i in range(8)]) parser.add_argument('--config', type=str, default='config.yaml') args = parser.parse_args() config = yaml.safe_load(open(args.config, 'r', encoding='utf-8')) config['gpu'] = args.gpu if args.task == 'preprocess': from src.preprocess import preprocess preprocess(config) elif args.task == 'train': from src.train import train_language_model train_language_model(config) elif args.task == 'test': from src.test import test_language_model test_language_model(config) elif args.task == 'predict': from src.predict import predict predict(config) elif args.task == 'select': from src.select import select select(config) else: raise ValueError('argument --task error')
def detectPlatesInScene(imgOriginalScene): listOfPossiblePlates = [] height, width, numChannels = imgOriginalScene.shape imgGrayscaleScene = np.zeros((height, width, 1), np.uint8) imgThreshScene = np.zeros((height, width, 1), np.uint8) imgContours = np.zeros((height, width, 3), np.uint8) cv2.destroyAllWindows() if main.showSteps == True: cv2.imshow("0", imgOriginalScene) imgGrayscaleScene, imgThreshScene = preprocess.preprocess(imgOriginalScene) if main.showSteps == True: cv2.imshow("1a", imgGrayscaleScene) cv2.imshow("1b", imgThreshScene) listOfPossibleCharsInScene = findPossibleCharsInScene(imgThreshScene) if main.showSteps == True: print("step 2 - len(listOfPossibleCharsInScene) = " + str( len(listOfPossibleCharsInScene))) imgContours = np.zeros((height, width, 3), np.uint8) contours = [] for possibleChar in listOfPossibleCharsInScene: contours.append(possibleChar.contour) cv2.drawContours(imgContours, contours, -1, main.SCALAR_WHITE) cv2.imshow("2b", imgContours) listOfListsOfMatchingCharsInScene = detect_chars.findListOfListsOfMatchingChars(listOfPossibleCharsInScene) if main.showSteps == True: print("step 3 - listOfListsOfMatchingCharsInScene.Count = " + str( len(listOfListsOfMatchingCharsInScene))) imgContours = np.zeros((height, width, 3), np.uint8) for listOfMatchingChars in listOfListsOfMatchingCharsInScene: intRandomBlue = random.randint(0, 255) intRandomGreen = random.randint(0, 255) intRandomRed = random.randint(0, 255) contours = [] for matchingChar in listOfMatchingChars: contours.append(matchingChar.contour) cv2.drawContours(imgContours, contours, -1, (intRandomBlue, intRandomGreen, intRandomRed)) cv2.imshow("3", imgContours) for listOfMatchingChars in listOfListsOfMatchingCharsInScene: possiblePlate = extractPlate(imgOriginalScene, listOfMatchingChars) if possiblePlate.imgPlate is not None: listOfPossiblePlates.append(possiblePlate) print("\n" + str(len(listOfPossiblePlates)) + " possible plates found") if main.showSteps == True: print("\n") cv2.imshow("4a", imgContours) for i in range(0, len(listOfPossiblePlates)): p2fRectPoints = cv2.boxPoints(listOfPossiblePlates[i].rrLocationOfPlateInScene) cv2.line(imgContours, tuple(p2fRectPoints[0]), tuple(p2fRectPoints[1]), main.SCALAR_RED, 2) cv2.line(imgContours, tuple(p2fRectPoints[1]), tuple(p2fRectPoints[2]), main.SCALAR_RED, 2) cv2.line(imgContours, tuple(p2fRectPoints[2]), tuple(p2fRectPoints[3]), main.SCALAR_RED, 2) cv2.line(imgContours, tuple(p2fRectPoints[3]), tuple(p2fRectPoints[0]), main.SCALAR_RED, 2) cv2.imshow("4a", imgContours) print("possible plate " + str(i) + ", click on any image and press a key to continue . . .") cv2.imshow("4b", listOfPossiblePlates[i].imgPlate) cv2.waitKey(0) print("\nplate detection complete, click on any image and press a key to begin char recognition . . .\n") cv2.waitKey(0) return listOfPossiblePlates
def detectCharsInPlates(listOfPossiblePlates): intPlateCounter = 0 imgContours = None contours = [] if len(listOfPossiblePlates) == 0: # if list of possible plates is empty return listOfPossiblePlates # return for possiblePlate in listOfPossiblePlates: # for each possible plate, this is a big for loop that takes up most of the function possiblePlate.imgGrayscale, possiblePlate.imgThresh = preprocess.preprocess( possiblePlate.imgPlate ) # preprocess to get grayscale and threshold images if main.showSteps == True: # show steps ################################################### cv2.imshow("5a", possiblePlate.imgPlate) cv2.imshow("5b", possiblePlate.imgGrayscale) cv2.imshow("5c", possiblePlate.imgThresh) possiblePlate.imgThresh = cv2.resize(possiblePlate.imgThresh, (0, 0), fx=1.6, fy=1.6) thresholdValue, possiblePlate.imgThresh = cv2.threshold( possiblePlate.imgThresh, 0.0, 255.0, cv2.THRESH_BINARY | cv2.THRESH_OTSU) if main.showSteps == True: # show steps ################################################### cv2.imshow("5d", possiblePlate.imgThresh) listOfPossibleCharsInPlate = findPossibleCharsInPlate( possiblePlate.imgGrayscale, possiblePlate.imgThresh) if main.showSteps == True: # show steps ################################################### height, width, numChannels = possiblePlate.imgPlate.shape imgContours = np.zeros((height, width, 3), np.uint8) del contours[:] # clear the contours list for possibleChar in listOfPossibleCharsInPlate: contours.append(possibleChar.contour) cv2.drawContours(imgContours, contours, -1, main.SCALAR_WHITE) cv2.imshow("6", imgContours) listOfListsOfMatchingCharsInPlate = findListOfListsOfMatchingChars( listOfPossibleCharsInPlate) if main.showSteps == True: imgContours = np.zeros((height, width, 3), np.uint8) del contours[:] for listOfMatchingChars in listOfListsOfMatchingCharsInPlate: intRandomBlue = random.randint(0, 255) intRandomGreen = random.randint(0, 255) intRandomRed = random.randint(0, 255) for matchingChar in listOfMatchingChars: contours.append(matchingChar.contour) cv2.drawContours(imgContours, contours, -1, (intRandomBlue, intRandomGreen, intRandomRed)) cv2.imshow("7", imgContours) if (len(listOfListsOfMatchingCharsInPlate) == 0): if main.showSteps == True: print( "chars found in plate number " + str(intPlateCounter) + " = (none), click on any image and press a key to continue . . ." ) intPlateCounter = intPlateCounter + 1 cv2.destroyWindow("8") cv2.destroyWindow("9") cv2.destroyWindow("10") cv2.waitKey(0) possiblePlate.strChars = "" continue for i in range(0, len(listOfListsOfMatchingCharsInPlate)): listOfListsOfMatchingCharsInPlate[i].sort( key=lambda matchingChar: matchingChar.intCenterX) listOfListsOfMatchingCharsInPlate[i] = removeInnerOverlappingChars( listOfListsOfMatchingCharsInPlate[i]) if main.showSteps == True: imgContours = np.zeros((height, width, 3), np.uint8) for listOfMatchingChars in listOfListsOfMatchingCharsInPlate: intRandomBlue = random.randint(0, 255) intRandomGreen = random.randint(0, 255) intRandomRed = random.randint(0, 255) del contours[:] for matchingChar in listOfMatchingChars: contours.append(matchingChar.contour) cv2.drawContours(imgContours, contours, -1, (intRandomBlue, intRandomGreen, intRandomRed)) cv2.imshow("8", imgContours) intLenOfLongestListOfChars = 0 intIndexOfLongestListOfChars = 0 for i in range(0, len(listOfListsOfMatchingCharsInPlate)): if len(listOfListsOfMatchingCharsInPlate[i] ) > intLenOfLongestListOfChars: intLenOfLongestListOfChars = len( listOfListsOfMatchingCharsInPlate[i]) intIndexOfLongestListOfChars = i longestListOfMatchingCharsInPlate = listOfListsOfMatchingCharsInPlate[ intIndexOfLongestListOfChars] if main.showSteps == True: imgContours = np.zeros((height, width, 3), np.uint8) del contours[:] for matchingChar in longestListOfMatchingCharsInPlate: contours.append(matchingChar.contour) cv2.drawContours(imgContours, contours, -1, main.SCALAR_WHITE) cv2.imshow("9", imgContours) possiblePlate.strChars = recognizeCharsInPlate( possiblePlate.imgThresh, longestListOfMatchingCharsInPlate) if main.showSteps == True: print("chars found in plate number " + str(intPlateCounter) + " = " + possiblePlate.strChars + ", click on any image and press a key to continue . . .") intPlateCounter = intPlateCounter + 1 cv2.waitKey(0) if main.showSteps == True: print( "\nchar detection complete, click on any image and press a key to continue . . .\n" ) cv2.waitKey(0) return listOfPossiblePlates
import cv2 image = X_train[6837] Image.fromarray(image).save("writeup/before.png") yuv = (255 * preprocess_image(image)).astype('uint8') Image.fromarray(yuv[:, :, 0]).save("writeup/y.png") Image.fromarray(yuv[:, :, 1]).save("writeup/u.png") Image.fromarray(yuv[:, :, 2]).save("writeup/v.png") rgb = cv2.cvtColor(yuv, cv2.COLOR_YUV2RGB) Image.fromarray(rgb).save("writeup/after.png") for i in np.arange(23205, 23210): im = Image.fromarray(X_train[i]) im.save("writeup/yield/" + str(i) + ".png") X_train, y_train = preprocess(X_train, y_train) X_valid, y_valid = preprocess(X_valid, y_valid) from sklearn.utils import shuffle shuffle(X_train, y_train) # Step 2B: Architecture from src.architecture import LeNet # Step 2C: Train the data import tensorflow as tf from sklearn.utils import shuffle EPOCHS = 20 BATCH_SIZE = 4096
logging.basicConfig(filename='log/log_preprocess.log', format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) parser = argparse.ArgumentParser() if __name__ == '__main__': parser.add_argument('corpus', choices=['def', 'ex', 'both'], help='choose type of corpus to preprocess') parser.add_argument('-pos', '--export_pos', action='store_true', help='export postagging') parser.add_argument( '-fn', '--is_final', action='store_true', help='preprocess for final model (not split to train/test)') args = parser.parse_args() if args.corpus in ['def', 'both']: preprocess.preprocess(CORPUS_FILES, skip_viTokenizer=False, export_pos=args.export_pos, is_final=args.is_final) if args.corpus in ['ex', 'both']: preprocess.preprocess(EXTERNAL_CORPUS_FILES, skip_viTokenizer=True, export_pos=args.export_pos, is_final=args.is_final)
def train(opt): traindata, trainGenerator, classes = preprocess(path='./data' + os.sep + opt.train, batchsize=opt.batchsize, imagesize=opt.imagesize, shuffle=True) valdata, validationGenerator, classes = preprocess(path='./data' + os.sep + opt.val, batchsize=opt.batchsize, imagesize=opt.imagesize, shuffle=True) # print(iter(trainGenerator).__next__()) num_channels = iter(trainGenerator).__next__()[0].size()[1] if opt.conv1_1 and opt.depth == 16: path_t = 'results/VdcnnIR_train_C11_{}.txt'.format(opt.depth) path_v = 'results/VdcnnIR_val_C11_{}.txt'.format(opt.depth) else: path_t = 'results/VdcnnIR_train_{}.txt'.format(opt.depth) path_v = 'results/VdcnnIR_val_{}.txt'.format(opt.depth) if os.path.exists(path_t): os.remove(path_t) os.mknod(path_t) else: os.mknod(path_t) if os.path.exists(path_v): os.remove(path_v) os.mknod(path_v) else: os.mknod(path_v) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if opt.conv1_1 and opt.depth == 16: model = Vgg(num_channels=num_channels, num_classes=classes, depth=opt.depth, initialize_weights=True, conv1_1=opt.conv1_1).to(device) else: model = Vgg(num_channels=num_channels, num_classes=classes, depth=opt.depth, initialize_weights=True, conv1_1=False).to(device) # return model optimizer = optim.Adam(model.parameters(), lr=opt.lr) criterion = nn.CrossEntropyLoss() def plot_fig(train_loss, val_loss): plt.figure(figsize=(10, 8)) if opt.conv1_1 and opt.depth == 16: plt.title("{}_C11 layer model".format(opt.depth)) else: plt.title("{} layer model".format(opt.depth)) plt.plot(train_loss, label='Train_loss') plt.plot(val_loss, label='Val_loss') plt.xlabel("Epochs") plt.ylabel("Loss") plt.legend() if opt.conv1_1 and opt.depth == 16: plt.savefig("figures/trainVal_loss_C11_{}.png".format(opt.depth)) else: plt.savefig("figures/trainVal_loss_{}.png".format(opt.depth)) # plt.show() # plt.close() return None totalVal_loss = [] totalTrain_loss = [] early_stop = False count = 0 best_score = None for epoch in range(opt.epochs): model.train() train_loss = [] total_predictions = [] total_labels = [] for idx, data in enumerate(trainGenerator): data_, label = data[0], data[1] data_ = data_.to(device) label = label.to(device) # print(data_.size()) optimizer.zero_grad() prob = model(data_) # print(prob) prob_ = np.argmax(prob.detach().cpu(), -1) loss = criterion(prob, label) train_loss.append(loss.item() * len(label.cpu())) loss.backward() optimizer.step() total_predictions.extend(prob_) total_labels.extend(label.cpu()) print('Iter: [{}/{}]\t Epoch: [{}/{}]\t Loss: {}\t Acc: {}'.format( idx + 1, len(trainGenerator), epoch + 1, opt.epochs, loss.item(), metrics.accuracy_score(label.cpu(), prob_))) loss_epoch = sum(train_loss) / len(traindata) totalTrain_loss.append(loss_epoch) with open(path_t, 'a') as f: f.write('Epoch: {}\t Loss: {}\t Accuracy: {}\n'.format( epoch + 1, loss_epoch, metrics.accuracy_score(total_labels, total_predictions))) model.eval() val_loss = [] total_Valpredictions = [] total_ValLabels = [] for idx_e, data_e in enumerate(validationGenerator): data_e, label_e = data_e[0], data_e[1] data_e = data_e.to(device) label_e = label_e.to(device) with torch.no_grad(): prob_e = model(data_e) loss_v = criterion(prob_e, label_e) pred_e = np.argmax(prob_e.detach().cpu(), -1) val_loss.append(loss_v.item() * len(label_e.cpu())) total_ValLabels.extend(label_e.cpu()) total_Valpredictions.extend(pred_e) print('Iter: [{}/{}]\t Epoch: [{}/{}]\t Loss: {}\t Acc: {}'. format(idx_e + 1, len(validationGenerator), epoch + 1, opt.epochs, loss_v.item(), metrics.accuracy_score(label_e.cpu(), pred_e))) val_lossEpoch = sum(val_loss) / len(valdata) totalVal_loss.append(val_lossEpoch) with open(path_v, 'a') as f: f.write('Epoch: {}\tLoss: {}\tAccuracy: {}\n'.format( epoch + 1, val_lossEpoch, metrics.accuracy_score(total_ValLabels, total_Valpredictions))) # roc_fig = scikitplot.metrics.plot_roc(total_ValLabels, total_Valpredictions, figsize=(12, 12)) # plt.savefig('figures/ROC_{}.png'.format(opt.depth)) # plt.show() plot_fig(train_loss=totalTrain_loss, val_loss=totalVal_loss) # print(loss_fig) if best_score is None: best_score = val_lossEpoch if opt.conv1_1 and opt.depth == 16: torch.save(model, 'models/VdcnnIR_C11_{}'.format(opt.depth)) else: torch.save(model, 'models/VdcnnIR_{}'.format(opt.depth)) elif val_lossEpoch > best_score: print("Loss:{} doesn't decreased from {}".format( val_lossEpoch, best_score)) count += 1 if count >= opt.early_stopping: early_stop = True elif val_lossEpoch < best_score: print("Loss:{} decreased from {}. Saving model........".format( val_lossEpoch, best_score)) best_score = val_lossEpoch if opt.conv1_1 and opt.depth == 16: torch.save(model, 'models/VdcnnIR_C11_{}'.format(opt.depth)) else: torch.save(model, 'models/VdcnnIR_{}'.format(opt.depth)) count = 0 if early_stop: break model.train() losses = {'trainLoss': totalTrain_loss, 'valLoss': totalVal_loss} if opt.conv1_1 and opt.depth == 16: with open('results/losses_C11_{}'.format(opt.depth), 'wb') as f: pickle.dump(losses, f) else: with open('results/losses_{}'.format(opt.depth), 'wb') as f: pickle.dump(losses, f) return best_score