default="", help="dataset location") # Log parser.add_argument("--log_path", type=str, default="./preprocess_pdtb.log", help="log path of pdtb output") args = parser.parse_args() if args.processed_dir.endswith(os.sep): args.processed_dir = args.processed_dir[:-1] args.encoder = args.encoder.lower() if args.encoder == "lstm": tokenizer = Word2VecTokenizer(**vars(args)) elif args.encoder == "bert": tokenizer = BERTTokenizer(**vars(args)) elif args.encoder == "roberta": tokenizer = ROBERTATokenizer(**vars(args)) else: raise NotImplementedError("Error: encoder=%s is not supported now." % (args.encoder)) encode_func = partial(tokenizer.encode, return_tensors=True) # Dataset pdtb_dataset = Dataset(data=None, encode_func=encode_func) pdtb_dataset.load_csv(args.csv_file_path, args.processed_dir, args.sections, args.types) pdtb_dataset.save_pt(args.dataset_file_path)
print("\nParameters:") if tf.__version__ < 1.5: FLAGS._parse_flags() # tf version <= 1.4 for attr, value in sorted(FLAGS.__flags.items()): print("{}={}".format(attr.upper(), value)) else: for attr in FLAGS: value = FLAGS[attr].value print("{}={}".format(attr.upper(), value)) print("") if not FLAGS.data_file: exit("Train data file is empty. Set --data_file argument.") dataset = Dataset(data_file=FLAGS.data_file, char_level=FLAGS.char_model, embedding_dim=FLAGS.embedding_dim) vocab, word2id = dataset.read_vocab() print("Vocabulary Size: {:d}".format(len(vocab))) def train(): with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): if FLAGS.model_class == 'siamese': model = SiameseNets(
n_hidden = args.n_hidden model_param = make_model_param() model_param['n_hidden'] = n_hidden if args.diagRNN: model_param['cell_type'] = 'diagLSTM' else: model_param['cell_type'] = 'LSTM' print "Computation start : " + str(datetime.now()) files_to_exclude = [] data = Dataset() data.load_data(args.data_path, note_range=note_range, timestep_type=timestep_type, max_len=max_len, exclude=files_to_exclude) save_path = args.save_path ckpt_path = os.path.join("ckpt", save_path) safe_mkdir(ckpt_path) model = make_model_from_dataset(data, model_param) result, res_dict = get_best_eval_metrics(data, model, save_path,
def main(): # Read dataset #dataset = Dataset('highway',1050, 1350) #dataset = Dataset('fall', 1461, 1560) dataset = Dataset('traffic', 951, 1050) imgs = dataset.readInput() imgs_GT = dataset.readGT() # cap = cv2.VideoCapture('source.mp4') kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) fgbg = createBackgroundSubtractor() substracted = [] gif_frames = [] fig = plt.figure() for i in range(len(imgs)): frame = imgs[i] fgmask = fgbg.apply(frame) fgmask = cv2.morphologyEx(fgmask, cv2.MORPH_OPEN, kernel) substracted.append(fgmask) if gif is True: g = plt.imshow(fgmask, cmap='gray', animated=True) plt.axis("off") gif_frames.append([g]) if showFrames is True: cv2.imshow('input', imgs[i]) cv2.imshow('frame', fgmask) k = cv2.waitKey(30) & 0xff if k == 27: break if PR_curve is True: precision, recall, auc_val = ev.getPR_AUC(imgs_GT, substracted) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, step='post', alpha=0.2, color='g') plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.0]) plt.xlim([0.0, 1.0]) plt.title("Precision-Recall curve - Fall - MOG2") plt.show() print len(precision) print 'precision' print sum(precision) / len(precision) print 'recall' print recall print 'Area under the curve ' print auc_val if Metrics is True: metrics = ev.getMetrics(imgs_GT, substracted) print metrics if gif is True: anim = animation.ArtistAnimation(fig, gif_frames, interval=len(imgs), blit=True) anim.save('animation.gif', writer='imagemagick', fps=10) # plt.show() #cap.release() cv2.destroyAllWindows()
from dataset import Dataset d = Dataset() passengers = d.getPassengers() print(passengers)
type=bool, help="Augment data.") args = parser.parse_args() args.logdir = os.path.join( "logs", "{}-{}-{}".format( os.path.basename(__file__), datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S"), ",".join( ("{}={}".format(re.sub("(.)[^_]*_?", r"\1", key), value) for key, value in sorted(vars(args).items()))))) print( "=========================================================================================" ) print("Loading data...") dataset = Dataset() print("Done.") print( "=========================================================================================" ) # Data info print( "=========================================================================================" ) print("Data info") print( "*****************************************************************************************" ) print("Training set size: {}".format(dataset.train.size)) print( "*****************************************************************************************"
import os import torch import torch.nn as nn from torchvision import transforms from dataset import Dataset from architectures.architecture import UNet_G, PatchGan_D_70x70 from trainers.trainer import Trainer from utils import save, load train_dir_name = ['data/file/train/input', 'data/file/train/target'] val_dir_name = ['data/file/val/input', 'data/file/val/target'] lr_D, lr_G, bs = 0.0002, 0.0002, 8 sz, ic, oc, use_sigmoid = 256, 3, 3, False norm_type = 'instancenorm' train_data = Dataset(train_dir_name, basic_types = 'Pix2Pix', shuffle = True) val_data = Dataset(val_dir_name, basic_types = 'Pix2Pix', shuffle = False) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') netD = PatchGan_D_70x70(ic, oc, use_sigmoid, norm_type).to(device) netG = UNet_G(ic, oc, sz, nz = 8, norm_type = norm_type).to(device) trainer = Trainer('SGAN', netD, netG, device, train_data, val_data, lr_D = lr_D, lr_G = lr_G, rec_weight = 10, ds_weight = 8, resample = True, weight_clip = None, use_gradient_penalty = False, loss_interval = 150, image_interval = 300, save_img_dir = 'saved_imges') trainer.train([5, 5, 5, 5], [0.5, 0.5, 0.5], [16, 16, 16, 16]) save('saved/cur_state.state', netD, netG, trainer.optimizerD, trainer.optimizerG)
def train(): vocab = read_vocab(FLAGS.vocab_data) glove = load_glove("glove.6B.{}d.txt".format(FLAGS.emb_size), FLAGS.emb_size, vocab) train = Dataset(filepath=FLAGS.train_data) valid = Dataset(filepath=FLAGS.valid_data) with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): han = HieAttNet(cell_type=FLAGS.cell_type, hid_size=FLAGS.hid_size, att_size=FLAGS.att_size, vocab_size=len(vocab), emb_size=FLAGS.emb_size, num_classes=FLAGS.num_classes, pretrained_embs=glove, l2_reg_lambda=FLAGS.l2_reg_lambda) # Define training procedure global_step = tf.Variable(0, name="global_step", trainable=False) train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize( han.loss, global_step=global_step) acc, acc_op = tf.metrics.accuracy(labels=han.labels, predictions=han.predictions, name="metrics/acc") metrics_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="metrics") metrics_init_op = tf.variables_initializer(var_list=metrics_vars) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", han.loss) acc_summary = tf.summary.scalar("accuracy", han.accuracy) # Train summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Valid summaries valid_step = 0 valid_summary_op = tf.summary.merge([loss_summary, acc_summary]) valid_summary_dir = os.path.join(out_dir, "summaries", "valid") valid_summary_writer = tf.summary.FileWriter( valid_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # initialize all variables best_valid_acc = 0.0 sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # training and validating loop for epoch in range(FLAGS.num_epochs): print('-' * 100) print('\n{}> epoch: {}\n'.format( datetime.datetime.now().isoformat(), epoch)) sess.run(metrics_init_op) # Training process for batch in train.bacth_iter(FLAGS.batch_size, desc="Training", shuffle=True): labels, docs = zip(*batch) padded_docs, sent_length, max_sent_length, word_length, max_word_length = normalize( docs) feed_dict = { han.docs: padded_docs, han.labels: labels, han.sent_length: sent_length, han.word_length: word_length, han.max_sent_length: max_sent_length, han.max_word_length: max_word_length, han.is_training: True, han.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy, _ = sess.run([ train_op, global_step, train_summary_op, han.loss, han.accuracy, acc_op ], feed_dict) train_summary_writer.add_summary(summaries, step) # training log display # if step % FLAGS.display_every == 0: # time_str = datetime.datetime.now().isoformat() # print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy)) # Model checkpoint # if step % FLAGS.checkpoint_every == 0: # path = saver.save(sess, checkpoint_prefix, global_step=step) # print("saved model checkpoint to {}\n".format(path)) print("\ntraining accuracy = {:.2f}\n".format( sess.run(acc) * 100)) sess.run(metrics_init_op) # Validating process for batch in valid.bacth_iter(FLAGS.batch_size, desc="Validating", shuffle=False): valid_step += 1 labels, docs = zip(*batch) padded_docs, sent_length, max_sent_length, word_length, max_word_length = normalize( docs) feed_dict = { han.docs: padded_docs, han.labels: labels, han.sent_length: sent_length, han.max_sent_length: max_sent_length, han.word_length: word_length, han.max_word_length: max_word_length, han.is_training: False, han.dropout_keep_prob: 1.0 } summaries, loss, accuracy, _ = sess.run( [valid_summary_op, han.loss, han.accuracy, acc_op], feed_dict) valid_summary_writer.add_summary(summaries, global_step=valid_step) valid_acc = sess.run(acc) * 100 print("\nvalidating accuracy = {:.2f}\n".format(valid_acc)) print("previous best validating accuracy = {:.2f}\n".format( best_valid_acc)) # model checkpoint if valid_acc > best_valid_acc: best_valid_acc = valid_acc path = saver.save(sess, checkpoint_prefix) print("saved model checkpoint to {}\n".format(path)) print("{} optimization finished!\n".format( datetime.datetime.now())) print("best validating accuracy = {:.2f}\n".format(best_valid_acc))
import sys import json from sklearn.ensemble import RandomForestClassifier from dataset import Dataset from utils import * if __name__ == "__main__": dataset = Dataset("../data") train_x, train_y, _, _ = dataset.get_cancel_data(onehot_x=True) print(train_x.shape) groups = dataset.get_groups("train") model = RandomForestClassifier(random_state=0, n_jobs=4) params_grid = { "criterion": ["entropy"], "n_estimators": [300], "max_features": ["auto", None], "max_depth": [8, 10, 20, 40, 60, 80, 100, None], "min_samples_leaf": [2, 3, 5, 8, 1e-4, 2e-4, 5e-4, 1e-3, 2e-3, 5e-3], } print(params_grid) #splits = range(2, 5) #cv = GroupTimeSeriesSplit(n_splits=5).split(train_x, groups=groups, select_splits=splits) split_groups = ['-'.join(g.split('-')[:2]) for g in groups] cv = sliding_monthly_split(train_x, split_groups=split_groups, start_group="2016-05", group_window=5, step=2, soft=True) results = single_search_cv(x=train_x, y=train_y, model=model, params_grid=params_grid, cv=cv, \
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=15, gamma=0.1) for k in [0, 1]: if k == 0: seq_train = sequences[:-val_len] seq_val = sequences[-val_len:] else: seq_train = sequences[val_len:] seq_val = sequences[:val_len] # use our dataset and defined transformations dataset_train = Dataset(DATA_DIR, transforms=get_transform(train=True), sequences=seq_train) dataset_val = Dataset(DATA_DIR, transforms=get_transform(train=False), sequences=seq_val) # define training and validation data loaders data_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers, collate_fn=utils.collate_fn) data_loader_val = torch.utils.data.DataLoader( dataset_val, batch_size=batch_size,
def main(): if len(sys.argv) != 4: print('Usage: python finetune.py train_file test_file weight_file') return train_file = sys.argv[1] test_file = sys.argv[2] weight_file = sys.argv[3] # Learning params learning_rate = 0.001 training_iters = 12800 batch_size = 50 display_step = 20 test_step = 640 # Network params n_classes = 20 keep_rate = 0.5 # Graph input x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, n_classes]) keep_var = tf.placeholder(tf.float32) # Model pred = Model.alexnet(x, keep_var) # Loss and optimizer loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y)) optimizer = tf.train.GradientDescentOptimizer( learning_rate=learning_rate).minimize(loss) # Evaluation correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Init init = tf.global_variables_initializer() # Load dataset dataset = Dataset(train_file, test_file) # Launch the graph with tf.Session() as sess: print('Init variable') sess.run(init) print('Load pre-trained model: {}'.format(weight_file)) load_with_skip(weight_file, sess, ['fc8']) # Skip weights from fc8 print('Start training') step = 1 while step < training_iters: batch_xs, batch_ys = dataset.next_batch(batch_size, 'train') sess.run(optimizer, feed_dict={ x: batch_xs, y: batch_ys, keep_var: keep_rate }) # Display testing status if step % test_step == 0: test_acc = 0. test_count = 0 for _ in range(dataset.test_size // batch_size): batch_tx, batch_ty = dataset.next_batch(batch_size, 'test') acc = sess.run(accuracy, feed_dict={ x: batch_tx, y: batch_ty, keep_var: 1. }) test_acc += acc test_count += 1 test_acc /= test_count print('{} Iter {}: Testing Accuracy = {:.4f}'.format( datetime.now(), step, test_acc), file=sys.stderr) # Display training status if step % display_step == 0: acc = sess.run(accuracy, feed_dict={ x: batch_xs, y: batch_ys, keep_var: 1. }) batch_loss = sess.run(loss, feed_dict={ x: batch_xs, y: batch_ys, keep_var: 1. }) print('{} Iter {}: Training Loss = {:.4f}, Accuracy = {:.4f}'. format(datetime.now(), step, batch_loss, acc), file=sys.stderr) step += 1 print('Finish!')
parser.add_argument('--ckpt_dir', default='results', type=str, help='The path of the checkpoint directory') args = parser.parse_args() config = 'z-{}_batch-{}_num-train-steps-{}'.format(args.latent_dim, args.batch_size, args.num_training_steps) args.ckpt_dir = os.path.join(args.ckpt_dir, config) args.log_dir = os.path.join(args.log_dir, config) device = torch.device( 'cuda' if torch.cuda.is_available() and not args.no_cuda else 'cpu') dataset = Dataset(args.batch_size, args.data_dir) model = VAE(1, args.latent_dim).to(device) tb_writer = SummaryWriter(args.log_dir) if args.do_train: optimizer = optim.Adam(model.parameters(), lr=args.learning_rate) trainer = Trainer(device, model, optimizer, dataset, args.ckpt_dir, tb_writer) trainer.train(args.num_training_steps, args.logging_steps, args.saving_steps) restore_ckpt_path = os.path.join( args.ckpt_dir, str(max(int(step) for step in os.listdir(args.ckpt_dir)))) model.restore(restore_ckpt_path)
def main(): try: opt_pairs, args = getopt.getopt( sys.argv[1:], "hd:k:i:m:", ["help", "dataset=", "nr_clusters=", "ip_type=", "model="]) except getopt.GetoptError, err: print str(err) usage() sys.exit(1) kwargs = {} for opt, arg in opt_pairs: if opt in ("-h", "--help"): usage() sys.exit(0) elif opt in ("-d", "--dataset"): src_cfg = arg elif opt in ("-k", "--nr_clusters"): nr_clusters = int(arg) elif opt in ("-m", "--model"): kwargs["model_type"] = arg elif opt in ("-i", "--ip_type"): ip_type = arg dataset = Dataset(src_cfg, nr_clusters=nr_clusters, ip_type=ip_type) evaluate_given_dataset(dataset, **kwargs) if __name__ == '__main__': main()
help='whether shuffle the dataset') parser.add_argument('-c', type=int, default=1000, help='number of class') parser.add_argument('-data_dir', type=str, default=None, help='path of the data') parser.add_argument('-pretrain_path', type=str, default=None, help='path of the pretrain model') args = parser.parse_args() net = get_network(args, num_classes=args.c) test_dataset = [ Dataset(args.data_dir, mode='test', pipeline=p) for p in test_pipeline ] test_loader = [ DataLoader(d, num_workers=args.w, batch_size=args.b, shuffle=args.s) for d in test_dataset ] net.load_state_dict(torch.load(args.weights), args.gpu) net.eval() result = np.zeros((len(test_dataset[0]), len(test_dataset), args.c)) with torch.no_grad(): for i, loader in enumerate(test_loader): for n_iter, (images, idxs) in enumerate(loader): images = images.cuda() pred = net(images).cpu().numpy()
"Maximum length of documents (default: 300)") tf.flags.DEFINE_string("checkpoint_dir", "runs/1596033083/checkpoints", "Checkpoint directory from training run") # Misc Parameters tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement") tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices") FLAGS = tf.flags.FLAGS print("Evaluating...\n") # Load test data test = Dataset(filepath=FLAGS.test_data, num_class=FLAGS.num_class, sequence_length=FLAGS.sequence_length) # Evaluation # ================================================== checkpoint_file = tf.compat.v1.train.latest_checkpoint(FLAGS.checkpoint_dir) graph = tf.Graph() with graph.as_default(): session_conf = tf.compat.v1.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.compat.v1.Session(config=session_conf) with sess.as_default(): # Load the saved meta graph and restore variables saver = tf.compat.v1.train.import_meta_graph( "{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file)
def main(params): timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") params['timestamp'] = timestamp add_to_report('# Training Report', params) add_to_report('\n## Parameters', params) add_to_report(params, params) print("Loading Dataset ...") # Initialize the datasets ds_train = Dataset(DATASET_DIR, 'train') ds_val = Dataset(DATASET_DIR, 'val') # Get the class names for this dataset class_names = ds_train.label_names mb_train, mb_val, tform = get_train_mb(ds_train, ds_val, params=params) if params.get('use_class_weights', False): print("Using class weights") add_to_report("\nClass weights", params) params['class_weight'] = get_class_weight(class_names, ds_train.classes()) for c, w in params['class_weight'].items(): print(" [%s] %f" % (class_names[c], w)) add_to_report(" - %s %f" % (class_names[c], w), params) # Initialize a softmax classifier print("Initializing CNN and optimizer ...") model = get_levinet(params) add_model_to_report(model, params) # SGD with Nesterov Momentum learning_rate = params.get('learning_rate', 1e-2) opt = SGD(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) try: print("Training for %i epochs ..." % params.get('n_epochs')) train_loop(model, mb_train, mb_val, opt=opt, params=params) except KeyboardInterrupt: print("\n\n*** Stopping Training\n\n") print("Testing best model on test set ...") # Initialize test data batchsize = params.get('batchsize', 64) ds_test = Dataset(DATASET_DIR, 'test') mb_test = MiniBatchGenerator(ds_test, batchsize, tform) print(" [%s] %i samples, %i minibatches of size %i" % (ds_test.split, mb_test.dataset.size(), mb_test.nbatches(), mb_test.batchsize())) # Load the global best model model = load_model('results/%s/best_model.h5' % timestamp, custom_objects={"LRN": LRN}) test_loop(model, mb_test, params=params)
W = np.asarray(val[1][0]) b = np.asarray(val[1][1]) if val[0] == 'conv1_1': W = W[:,:,0:1,:] if val[0] == 'conv5_1': break print(" Loading %s: %s, %s" % (val[0], W.shape, b.shape)) params.extend([W, b]) tl.files.assign_params(sess, params, self.net_vgg) ''' if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('-c', '--ckpt', type=int, default=0, help='') args = parser.parse_args() training_dataset = Dataset(config.TRAIN.target3d_path, config.TRAIN.lf2d_path, n_slices, n_num, base_size, normalize_mode=normalize_mode) trainer = Trainer(training_dataset) trainer.build_graph() trainer.train(begin_epoch=args.ckpt)
default=False, action='store_true', help='load checkpoint') parser.add_argument('--save_path', default='/home/share/new_vgg10/ASD-PB/', type=str, help='path to save checkpoint') parser.add_argument('--gpu', default=0, type=int, help='gpu id') parser.add_argument('--log_path', default='./logs', type=str, help='path to log') args = parser.parse_args() train_dataset = Dataset(args.data_path, args.dataset, True) train_loader = data.DataLoader(train_dataset, batch_size=args.bs, shuffle=True) test_dataset = Dataset(args.data_path, args.dataset, False) test_loader = data.DataLoader(test_dataset, batch_size=1, shuffle=False) device = torch.device('cuda:' + str(args.gpu)) model = Model().to(device) #writer = SummaryWriter(args.log_path) mseloss = nn.MSELoss(reduction='sum').to(device) bceloss = nn.BCELoss(reduction='sum').to(device) optimizer = optim.Adam(model.parameters(), lr=1e-4) if args.load:
def parse_prism(folder, filename): assert ('mdps' in folder) assert ('.prism' in filename) def nondigits_to_float(token): if (token == 'false'): return 0.0 if (token == 'true'): return 1.0 assert (token[0] == '-') return -(float(token[1:])) data = False X = [] Y = [] Xnames = [] Xdomains = [] Ynames = {'no': 0, 'yes': 1} Actions = {} # map string(action name)->int(action id) ActionIDtoName = {} # map int(action id)->string(action name) Modules = {'synchronous': 0} # map string->int ModuleIDtoName = {0: 'synchronous'} # map int->string ModActPlayed = set() # First scan to collect actions played, names and ranges for line in open('%s/%s' % (folder, filename), 'r'): if line.startswith('#'): continue # first line if line.startswith('\n'): break # we have finished parsing if line.startswith('('): # sample - get action played assert (len(Xnames) and len(Xdomains)) [_, act] = line.split(':') act = act.strip() mod = 'synchronous' if ('async_' in act): assert (len(act.split('_')) > 2) # async actionname modulename mod = act.split('_')[-1] act = '_'.join(act.split('_')[:-1]) if act not in Actions: pos = len(Actions) Actions[act] = pos ActionIDtoName[pos] = act if mod not in Modules: pos = len(Modules) Modules[mod] = pos ModuleIDtoName[pos] = mod if (mod, act) not in ModActPlayed: ModActPlayed.add((mod, act)) else: assert (not len(Xnames) and not len(Xdomains)) namesRanges = line.split(']')[:-1] assert (len(namesRanges) > 1) for nr in namesRanges: [name, rnge] = nr.split('[') if ';' in name: name = name[name.index(';') + 1:] name = name[:-1] if ('_' in name and len(name.split('_')) == 2 and len(name.split('_')[1]) > 1): name = name.split('_')[1] Xnames.append(name) [mi, ma] = rnge.split(',') mi, ma = int(mi), int(ma) assert (mi < ma) Xdomains.append(set({e for e in range(mi, ma + 1)})) # Add module and action features Xnames.append('module') Xdomains.append(set({e for e in range(len(Modules))})) Xnames.append('action') Xdomains.append(set({e for e in range(len(Actions))})) Xineqforbidden = set({len(Xnames) - 2, len(Xnames) - 1}) # Asserts allnames = set() for name in Xnames: assert (name not in allnames) allnames.add(name) assert (len(Actions) == len(ActionIDtoName)) for action in Actions: assert (ActionIDtoName[Actions[action]] == action) assert (len(Actions) >= 2) # Second scan to collect samples for line in open('%s/%s' % (folder, filename), 'r'): if line.startswith('#'): continue # first line if line.startswith('\n'): break # we have finished parsing if line.startswith('('): # sample assert (len(Xnames) and len(Xdomains)) [smp, act] = line.split(':') assert (smp[0] == '(' and smp[-1] == ')') smp = smp[1:-1] sample = [ float(e) if e.isdigit() else nondigits_to_float(e) for e in smp.split(',') ] act = act.strip() mod = 'synchronous' if ('async_' in act): assert (len(act.split('_')) > 2) # async actionname modulename mod = act.split('_')[-1] act = '_'.join(act.split('_')[:-1]) assert (act in Actions) assert (mod in Modules) assert ((mod, act) in ModActPlayed) sample.append(float(Modules[mod])) sample.append(float(Actions[act])) Y.append(Ynames['yes']) X.append(sample) # copies for other (module,action)s played for (modX, actX) in sorted(ModActPlayed): if (modX != mod or actX != act): badsam = sample.copy() badsam[-2] = float(Modules[modX]) badsam[-1] = float(Actions[actX]) X.append(badsam) Y.append(Ynames['no']) return Dataset(np.array(X), np.array(Y), np.array(Xnames), Xdomains, Ynames, Xineqforbidden, ActionIDtoName, ModuleIDtoName)
total_loss = dice_loss + (1 * focal_loss) # actulally total_loss can be imported directly from library, above example just show you how to manipulate with losses # total_loss = sm.losses.binary_focal_dice_loss # or sm.losses.categorical_focal_dice_loss metrics = [ sm.metrics.IOUScore(threshold=0.5), sm.metrics.FScore(threshold=0.5) ] # compile keras model with defined optimozer, loss and metrics model.compile(optim, total_loss, metrics) test_dataset = Dataset( x_test_dir, y_test_dir, classes=CLASSES, augmentation=get_validation_augmentation(), preprocessing=get_preprocessing(preprocess_input), ) test_dataloader = Dataloder(test_dataset, batch_size=1, shuffle=False) # load best weights model.load_weights('./saved_model/resnet18_Unet.h5') scores = model.evaluate_generator(test_dataloader) print("Loss: {:.5}".format(scores[0])) for metric, value in zip(metrics, scores[1:]): print("mean {}: {:.5}".format(metric.__name__, value)) # n = 5 # ids = np.random.choice(np.arange(len(test_dataset)), size=n)
parser.add_argument('-epochs', type=int, default=100) parser.add_argument('-model_path', default='models/test_model/') parser.add_argument('-log_path', default='tensorboard/') parser.add_argument('-log_name', default=None) parser.add_argument('-checkpoints', default=5) parser.add_argument('-v', type=int, default=0) parser.add_argument('-njobs', type=int, default=8) args = parser.parse_args() # Load the dataset. from dataset import Dataset dataset_train = Dataset(args.index, selector=args.train_selector, internal_shuffle=True, num_of_samples=args.n_train_samples, n_jobs=args.njobs, verbose=args.v) dataset_valid = Dataset(args.index, selector=args.valid_selector, internal_shuffle=True, num_of_samples=args.n_valid_samples, n_jobs=args.njobs, verbose=args.v) dataset_train.train_scaler(remove_mean=True, remove_std=True, scaler_batch_size=args.batch_size) dataset_valid.set_scaler(dataset_train.get_scaler()) print('Train with %d images' % (dataset_train.n_samples))
from torchvision import transforms from dataset import Dataset from architectures.unet import UNet_G from architectures.resnet import ResNet_G from architectures.discriminator import PatchGan_D_70x70_One_Input from trainers.trainer import Trainer from utils.utils import save, load train_dir_name = ['data/file/train/input', 'data/file/train/target'] val_dir_name = ['data/file/val/input', 'data/file/val/target'] lr_D, lr_G, bs = 0.0002, 0.0002, 8 sz, ic, oc, use_sigmoid = 256, 3, 3, False norm_type = 'instancenorm' train_data = Dataset(train_dir_name, basic_types = 'CycleGan', shuffle = True, single_channel = False) val_data = Dataset(val_dir_name, basic_types = 'Pix2Pix', shuffle = False, single_channel = False) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') netD_A = PatchGan_D_70x70_One_Input(ic, use_sigmoid, norm_type).to(device) netD_B = PatchGan_D_70x70_One_Input(oc, use_sigmoid, norm_type).to(device) netG_A2B = ResNet_G(ic, oc, sz, nz = 8, norm_type = norm_type).to(device) netG_B2A = ResNet_G(oc, ic, sz, nz = 8, norm_type = norm_type).to(device) trn_dl = train_data.get_loader(256, bs) val_dl = list(val_data.get_loader(256, 3))[0] trainer = Trainer('SGAN', netD_A, netD_B, netG_A2B, netG_B2A, device, trn_dl, val_dl, lr_D = lr_D, lr_G = lr_G, cycle_weight = 10, identity_weight = 5.0, ds_weight = 8, resample = True, weight_clip = None, use_gradient_penalty = False, loss_interval = 150, image_interval = 300, save_img_dir = 'saved_imges') trainer.train(5) save('saved/cur_state.state', netD_A, netD_B, netG_A2B, netG_B2A, trainer.optimizerD_A, trainer.optimizerD_B, trainer.optimizerG)
def classification(self, cls_type, ft, dataset, epochs, batch_size, learning_rate, max_seq_len, distil, alpha, temperature, text_column, label_column): ''' Loads datasets, models and runs training and evaluation pipelines :param cls_type: classifictaion task type :param ft: whether finetuning should be done :param dataset: dataset name :param epochs: number of training epochs :param batch_size: batch size :param learning_rate: learning rate :param max_seq_len: maximum sequence length :param distil: whether to distil model :param alpha: weight for student loss :param temperature: temperature for softmax operation for distillation :param text_column: column name for text in dataset :param label_column: column name for label in dataset ''' if ft: train_dataset = Dataset(dataset, split=Dataset.TRAIN_STR) val_dataset = Dataset(dataset, split=Dataset.VALIDATION_STR) for i in (self.model1, self.model2): opt = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0) loss_fn = tf.keras.losses.CategoricalCrossentropy( from_logits=False) metrics = [ 'accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.AUC() ] model = i.load_model( cls_type, train_dataset.get_num_classes(label_column=label_column), max_seq_len) model.compile(optimizer=opt, loss=loss_fn, metrics=metrics) if ft: tf_train_data = train_dataset.classification_tokenize( i.tokenizer, batch_size, max_seq_len, i.name, text_column=text_column, label_column=label_column) model.fit(tf_train_data, epochs=epochs) if distil: model_teacher = i.load_model( cls_type, train_dataset.get_num_classes( label_column=label_column), max_seq_len, for_distillation=True, temperature=temperature) model_teacher.compile(optimizer=opt, loss=loss_fn, metrics=metrics) model_teacher.fit(tf_train_data, epochs=epochs) model_soft_labels = model_teacher.predict(tf_train_data) student_dataset, encoder = train_dataset.student_dataset_encoder( model_soft_labels, batch_size, text_column=text_column, label_column=label_column) model_student = Model.student_model( cls_type, encoder, train_dataset.get_num_classes( label_column=label_column), temperature=temperature) student_loss_fn = { 'soft': Model.get_distillation_loss_fn(), 'hard': Model.get_distillation_loss_fn() } loss_wts = {'soft': 1 - alpha, 'hard': alpha} model_student.compile(optimizer=opt, loss=student_loss_fn, loss_weights=loss_wts, metrics=metrics) model_student.fit(student_dataset, epochs=epochs) del model_soft_labels del student_dataset del tf_train_data tf_val_data = val_dataset.classification_tokenize( i.tokenizer, batch_size, max_seq_len, i.name, text_column=text_column, label_column=label_column) start = time.time() model_eval = { k: v for k, v in zip(model.metrics_names, model.evaluate(tf_val_data, verbose=0)) } time_taken = time.time() - start self.results[cls_type][i.name] = model_eval try: self.results[cls_type][i.name]['f1'] = ( 2 * model_eval['precision'] * model_eval['recall']) / ( model_eval['precision'] + model_eval['recall']) except ZeroDivisionError: self.results[cls_type][i.name]['f1'] = 0 self.results[cls_type][i.name]['time taken'] = time_taken del self.results[cls_type][i.name]['precision'] del self.results[cls_type][i.name]['recall'] if distil: model_soft_labels = model.predict(tf_val_data) val_student_dataset, encoder = val_dataset.student_dataset_encoder( model_soft_labels, batch_size, text_column=text_column, label_column=label_column) start = time.time() model_eval = { k.split('_')[-1]: v for k, v in zip( model_student.metrics_names, model_student.evaluate(val_student_dataset, verbose=0)) if 'soft' not in k } time_taken = time.time() - start self.results[cls_type]['distilled-' + i.name] = model_eval try: self.results[cls_type]['distilled-' + i.name]['f1'] = ( 2 * model_eval['precision'] * model_eval['recall']) / ( model_eval['precision'] + model_eval['recall']) except ZeroDivisionError: self.results[cls_type]['distilled-' + i.name]['f1'] = 0 self.results[cls_type]['distilled-' + i.name]['time taken'] = time_taken del self.results[cls_type]['distilled-' + i.name]['precision'] del self.results[cls_type]['distilled-' + i.name]['recall'] del tf_val_data del val_student_dataset Model.clean_up()
def main(): # Load dataset print('Loading dataset ...\n') dataset_train = Dataset(root=opt.data, crop_size=(64, 64)) # dataset_val = Dataset(train=False) loader_train = DataLoader(dataset=dataset_train, num_workers=4, batch_size=opt.batch, shuffle=True) print('Dataset length:', len(loader_train)) print("# of training samples: %d\n" % int(len(dataset_train))) # Build model net = DnCNN(channels=1, num_of_layers=opt.num_of_layers) net.apply(weights_init_kaiming) criterion = nn.MSELoss(size_average=False) # Move to GPU device_ids = [0] model = nn.DataParallel(net, device_ids=device_ids).cuda() criterion.cuda() # Optimizer optimizer = optim.Adam(model.parameters(), lr=opt.lr) # training # writer = SummaryWriter(opt.outf) step = 0 # noiseL_B=[0,55] # ingnored when opt.mode=='S' for epoch in range(opt.epochs): if epoch < opt.milestone: current_lr = opt.lr else: current_lr = opt.lr / 10. # set learning rate for param_group in optimizer.param_groups: param_group["lr"] = current_lr print('learning rate %f' % current_lr) # train for i, data in enumerate(loader_train, 0): # training step model.train() model.zero_grad() optimizer.zero_grad() img_lr, img_hr = data img_lr, img_hr = img_lr.cuda(), img_hr.cuda() learned_noise = model(img_lr) loss = criterion(learned_noise, img_lr - img_hr) / (img_hr.size()[0] * 2) loss.backward() optimizer.step() # results model.eval() learned_img = torch.clamp(img_lr - learned_noise, 0., 1.) psnr_train = batch_PSNR(learned_img, img_hr, 1.) print( "[epoch %d][%d/%d] loss: %.4f PSNR_train: %.4f" % (epoch + 1, i + 1, len(loader_train), loss.item(), psnr_train)) # if you are using older version of PyTorch, you may need to change loss.item() to loss.data[0] # if step % 10 == 0: # # Log the scalar values # writer.add_scalar('loss', loss.item(), step) # writer.add_scalar('PSNR on training data', psnr_train, step) step += 1 ## the end of each epoch model.eval() # validate # psnr_val = 0 # for k in range(len(dataset_val)): # img_val = torch.unsqueeze(dataset_val[k], 0) # noise = torch.FloatTensor(img_val.size()).normal_(mean=0, std=opt.val_noiseL/255.) # imgn_val = img_val + noise # img_val, imgn_val = Variable(img_val.cuda(), volatile=True), Variable(imgn_val.cuda(), volatile=True) # out_val = torch.clamp(imgn_val-model(imgn_val), 0., 1.) # psnr_val += batch_PSNR(out_val, img_val, 1.) # psnr_val /= len(dataset_val) # print("\n[epoch %d] PSNR_val: %.4f" % (epoch+1, psnr_val)) # writer.add_scalar('PSNR on validation data', psnr_val, epoch) # log the images # out_train = torch.clamp(imgn_train-model(imgn_train), 0., 1.) # Img = utils.make_grid(img_train.data, nrow=8, normalize=True, scale_each=True) # Imgn = utils.make_grid(imgn_train.data, nrow=8, normalize=True, scale_each=True) # Irecon = utils.make_grid(out_train.data, nrow=8, normalize=True, scale_each=True) # writer.add_image('clean image', Img, epoch) # writer.add_image('noisy image', Imgn, epoch) # writer.add_image('reconstructed image', Irecon, epoch) # save model torch.save(model.state_dict(), os.path.join(opt.outf, 'net.pth'))
import argparse import numpy as np import torch import torch.nn as nn import torch.optim as optim from dataset import Dataset from dataset import DataLoader train_dataset = Dataset('data') loader = DataLoader(train_dataset) for batch_inputs, batch_outputs, masks, lengths in loader: loader.print_batch(batch_inputs) print(lengths) print(masks)
def get_dataloader(file_path, max_len, preprocessor, batch_size): dataset = Dataset(file_path, max_len, preprocessor) dataloader = DataLoader(dataset, batch_size=batch_size) return dataloader
train_params = {'batch_size': batch_size, 'shuffle': True, 'num_workers': 1} test_params = {'batch_size': 1, 'shuffle': False, 'num_workers': 1} max_epochs = 100 jar = open('data/glove_lstm/partition_dict.pkl', 'rb') partition = pickle.load(jar) jar2 = open('data/glove_lstm/label_dict.pkl', 'rb') labels = pickle.load(jar2) jar.close() jar2.close() # Generators training_set = Dataset(partition['train'], labels, 'glove_lstm') training_generator = torch.utils.data.DataLoader(training_set, **train_params) test_set = Dataset(partition['test'], labels, 'glove_lstm') test_generator = torch.utils.data.DataLoader(test_set, **test_params) # Models ffn = LSTMNetwork(100, layer_number, layer_width) ffn.cuda() # criterion = nn.BCELoss() criterion = nn.L1Loss() optimizer = optim.SGD(ffn.parameters(), lr=float(lr8), momentum=0.9) def test_loss():
def query_instances(args, unlabeled_dataset, oracle, active_func="random"): # lc stands for least confident # te stands for token entropy # tte stands for total token entropy assert active_func in [ "random", "longest", "shortest", "lc", "margin", "te", "tte" ] # lengths represents number of tokens, so BPE should be removed lengths = np.array([ len(remove_special_tok(remove_bpe(s)).split()) for s in unlabeled_dataset ]) # Preparations before querying instances # Reloading network parameters args.use_cuda = (args.no_cuda == False) and torch.cuda.is_available() net, _ = model.get() assert os.path.exists(args.checkpoint) net, src_vocab, tgt_vocab = load_model(args.checkpoint, net) if args.use_cuda: net = net.cuda() # Initialize inference dataset (Unlabeled dataset) infer_dataset = Dataset(unlabeled_dataset, src_vocab) if args.batch_size is not None: infer_dataset.BATCH_SIZE = args.batch_size if args.max_batch_size is not None: infer_dataset.max_batch_size = args.max_batch_size if args.tokens_per_batch is not None: infer_dataset.tokens_per_batch = args.tokens_per_batch infer_dataiter = iter( infer_dataset.get_iterator(shuffle=True, group_by_size=True, include_indices=True)) # Start ranking unlabeled dataset indices = np.arange(len(unlabeled_dataset)) if active_func == "random": result = get_scores(args, net, active_func, infer_dataiter, src_vocab, tgt_vocab) random.shuffle(result) indices = [item[1] for item in result] indices = np.array(indices).astype('int') for idx in indices: print("S:", unlabeled_dataset[idx]) print("H:", result[idx][2]) print("T:", oracle[idx]) print("V:", 0.0) print("I:", args.input, args.reference, idx) elif active_func == "longest": result = get_scores(args, net, active_func, infer_dataiter, src_vocab, tgt_vocab) result = [(len( remove_special_tok(remove_bpe( unlabeled_dataset[item[1]])).split(' ')), item[1], item[2]) for item in result] result = sorted(result, key=lambda item: -item[0]) indices = [item[1] for item in result] indices = np.array(indices).astype('int') for idx in indices: print("S:", unlabeled_dataset[idx]) print("H:", result[idx][2]) print("T:", oracle[idx]) print("V:", -result[idx][0]) print("I:", args.input, args.reference, idx) elif active_func == "shortest": result = get_scores(args, net, active_func, infer_dataiter, src_vocab, tgt_vocab) result = [(len( remove_special_tok(remove_bpe( unlabeled_dataset[item[1]])).split(' ')), item[1], item[2]) for item in result] result = sorted(result, key=lambda item: item[0]) indices = [item[1] for item in result] indices = np.array(indices).astype('int') for idx in indices: print("S:", unlabeled_dataset[idx]) print("H:", result[idx][2]) print("T:", oracle[idx]) print("V:", result[idx][0]) print("I:", args.input, args.reference, idx) indices = indices[np.argsort(lengths[indices])] elif active_func in ["lc", "margin", "te", "tte"]: result = get_scores(args, net, active_func, infer_dataiter, src_vocab, tgt_vocab) result = sorted(result, key=lambda item: item[0]) indices = [item[1] for item in result] indices = np.array(indices).astype('int') for idx in range(len(result)): print("S:", unlabeled_dataset[result[idx][1]]) print("H:", result[idx][2]) print("T:", oracle[result[idx][1]]) print("V:", result[idx][0]) print("I:", args.input, args.reference, result[idx][1])
def startLearning(): #Init Tensorboard writer = SummaryWriter() #Define batch size the number of epoch batch_size = 16 max_epochs = 50 #Make model model = AutoEncoder().cuda() #Define loss type criterion_expressions = nn.CrossEntropyLoss().cuda() criterion_landmarks = nn.MSELoss().cuda() #Define the optimizer optimizer = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999)) #Define the scheduler scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=5) best_acc = None print("load dataset") #Load Dataset loader_filenames = np.asarray([ "training_dataset_pack0.h5", "training_dataset_pack1.h5", "training_dataset_pack2.h5", "training_dataset_pack3.h5" ]) validation_loader = torch.utils.data.DataLoader(dataset=Dataset( 'validation_dataset_pack0.h5', "std_training.png", "mean_training.png"), batch_size=batch_size, shuffle=True, num_workers=1) print("Done") #Main loop (epoch) for epoch in range(1, max_epochs + 1): np.random.shuffle(loader_filenames) is_best = False print("Training...") #Init metrics loss_exp = 0. acc_exp = 0. loss_lm = 0. acc_lm = 0. expressions_loss = 0. landmarks_loss = 0. expressions_acc = 0. landmarks_acc = 0. count = 0 for loader_filename in loader_filenames: training_loader = torch.utils.data.DataLoader( dataset=Dataset(loader_filename, "std_training.png", "mean_training.png"), batch_size=batch_size, shuffle=True, num_workers=4) #Init progress bar for training pbart = tqdm.tqdm(total=int( len(training_loader.dataset) / batch_size), postfix={ "loss_e": None, "acc_e": None, "loss_l": None, "acc_l": None }, desc="Epoch: {}/{}".format(epoch, max_epochs)) #Training loop for i, data in enumerate(training_loader, 0): count += 1 #Zero the parameter gradients optimizer.zero_grad() #Get the inputs images, landmarks, expressions = data images = images.to(device) landmarks = landmarks.to(device).float() expressions = expressions.to(device).long() #Get the outputs outputs = model(images) #Calculate metrics #Loss expressions_loss = criterion_expressions( outputs[0], expressions) landmarks_loss = criterion_landmarks(outputs[1], landmarks) loss_lm += landmarks_loss.item() loss_exp += expressions_loss.item() #Accuracy _, predicted_expressions = torch.max(outputs[0], 1) expressions_acc += (predicted_expressions == expressions).sum().float() / batch_size acc_exp += (predicted_expressions == expressions).sum().float().item() / batch_size predicted_landmarks = outputs[1] #if epoch==2 and i ==5: # print(predicted_landmarks.detach().cpu().numpy().shape,landmarks.cpu().numpy().shape) # print(predicted_landmarks.detach().cpu().numpy()[0,:],landmarks.cpu().numpy()[0,:]) # print(np.abs(predicted_landmarks.detach().cpu().numpy()[0,:]-landmarks.cpu().numpy()[0,:])) landmarks_acc += 1 - (np.mean( np.abs(predicted_landmarks.detach().cpu().numpy() - landmarks.cpu().numpy())) / 128) acc_lm += 1 - (np.mean( np.abs(predicted_landmarks.detach().cpu().numpy() - landmarks.cpu().numpy())) / 128) #Backpropagation landmarks_loss.backward() expressions_loss.backward() #Reduce learning rate if we are on a plateau optimizer.step() #Update pbart.update(1) pbart.set_postfix({ "loss_e": loss_exp / count, "acc_e": expressions_acc.item() / count, "loss_l": loss_lm / count, "acc_l": landmarks_acc.item() / count }) pbart.close() #Calculate metrics on one epoch loss_exp /= (count) / batch_size acc_exp /= (count) / batch_size loss_lm /= (count) / batch_size acc_lm /= (count) / batch_size #Save metrics in a log file with open("log/training_DAN.log", "a") as f: f.write( "epoch: {} / {} loss_e: {} acc_e: {} loss_l: {} acc_l: {}\n". format(epoch, max_epochs, loss_exp, acc_exp, loss_lm, acc_lm)) f.close() #Construct tensorboard graph writer.add_scalar('data/Loss_expressions_training', loss_exp, epoch) writer.add_scalar('data/Accuracy_expressions_training', acc_exp, epoch) writer.add_scalar('data/Loss_landmarks_training', loss_lm, epoch) writer.add_scalar('data/Accuracy_landmarks_training', acc_lm, epoch) #Init metrics loss_exp = 0. acc_exp = 0. loss_lm = 0. acc_lm = 0. expressions_loss = 0. landmarks_loss = 0. expressions_acc = 0. landmarks_acc = 0. count = 0 print("Validation...") #Init progress bar for validation pbarv = tqdm.tqdm(total=int( len(validation_loader.dataset) / batch_size), postfix={ "loss_e": None, "acc_e": None, "loss_l": None, "acc_l": None }, desc="Epoch: {}/{}".format(epoch, max_epochs)) #Validation loop with torch.no_grad(): for i, data in enumerate(validation_loader, 0): count += 1 #Get the inputs images, landmarks, expressions = data images = images.to(device) landmarks = landmarks.to(device).float() expressions = expressions.to(device).long() #Get the outputs outputs = model(images) #Calculate metrics #Loss expressions_loss = criterion_expressions( outputs[0], expressions) landmarks_loss = criterion_landmarks(outputs[1], landmarks) loss_lm += landmarks_loss.item() loss_exp += expressions_loss.item() #Accuracy _, predicted_expressions = torch.max(outputs[0], 1) expressions_acc += (predicted_expressions == expressions).sum().float() / batch_size acc_exp += (predicted_expressions == expressions).sum().float().item() / batch_size predicted_landmarks = outputs[1] landmarks_acc += (1 - (np.mean( np.abs(predicted_landmarks.cpu().numpy() - landmarks.cpu().numpy())) / 128)) acc_lm += (1 - (np.mean( np.abs(predicted_landmarks.cpu().numpy() - landmarks.cpu().numpy())) / 128)) #Uptate validation progress bar pbarv.update(1) pbarv.set_postfix({ "loss_e": loss_exp / count, "acc_e": expressions_acc.item() / count, "loss_l": loss_lm / count, "acc_l": landmarks_acc.item() / count }) pbarv.close() #Calculate metrics on one epoch loss_exp /= (count) / batch_size acc_exp /= (count) / batch_size loss_lm /= (count) / batch_size acc_lm /= (count) / batch_size #Save the weights of the model if best_acc == None or acc_exp < best_acc: best_acc = acc_exp is_best = True save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'loss expressions': loss_exp, 'accuracy expressions': acc_exp, 'loss landmarks': loss_lm, 'accuracy landmarks': acc_lm, 'optimizer': optimizer.state_dict() }, is_best, "save_model/checkpoint_DAN.pth", "save_model/best_model_validation.pth") is_best = False scheduler.step(loss_lm) #Save metrics in a log file with open("log/validation_DAN.log", "a") as f: f.write( "epoch: {} / {} loss_e: {} acc_e: {} loss_l: {} acc_l: {}\n". format(epoch, max_epochs, loss_exp, acc_exp, loss_lm, acc_lm)) f.close() #Construct tensorboard graph writer.add_scalar('data/Loss_expressions_validation', loss_exp, epoch) writer.add_scalar('data/Accuracy_expressions_validation', acc_exp, epoch) writer.add_scalar('data/Loss_landmarks_validation', loss_lm, epoch) writer.add_scalar('data/Accuracy_landmarks_validation', acc_lm, epoch)
# 读取数据 num = pd.DataFrame(Dataset.load_part(name,'numeric'),columns = Dataset.get_part_feature('numeric')) # 初始化数据 df = pd.DataFrame(index=num.index) # 新增feature,数值型特征的减法计算 df['diff_1_6'] = num['cont1'] - num['cont6'] df['diff_1_9'] = num['cont1'] - num['cont9'] df['diff_1_10'] = num['cont1'] - num['cont10'] df['diff_6_9'] = num['cont6'] - num['cont9'] df['diff_6_10'] = num['cont6'] - num['cont10'] df['diff_6_11'] = num['cont6'] - num['cont11'] df['diff_6_12'] = num['cont6'] - num['cont12'] df['diff_6_13'] = num['cont6'] - num['cont13'] df['diff_7_11'] = num['cont7'] - num['cont11'] df['diff_7_12'] = num['cont7'] - num['cont12'] df['diff_11_12'] = num['cont11'] - num['cont12'] if name == 'train': Dataset.save_part_feature('numeric_combinations',list(df.columns)) Dataset(numeric_combinations=df.values).save(name) print ('Done.')