def main(cfg): torch.multiprocessing.set_sharing_strategy('file_system') seed_torch(seed=cfg.seed) output_dir = os.path.join(cfg.output_dir, cfg.desc) if not os.path.exists(output_dir): os.makedirs(output_dir) train_dataset = build_dataset(cfg, phase='train') test_dataset = build_dataset(cfg, phase='test') if cfg.DATA.weighted_sample: train_dl = DataLoader(train_dataset, batch_size=32, sampler=WeightedRandomSampler( train_dataset.get_label_weight(), num_samples=5000), num_workers=0, drop_last=True) else: train_dl = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=16, drop_last=True) test_dl = DataLoader(test_dataset, batch_size=32, num_workers=8, drop_last=True) solver = Solver(cfg) solver.train(train_dl, test_dl)
def main(): args = parse_args() log(args, '\n' + str(args)) if args.mode == 'train': comparision.train(args) elif args.mode == 'classify': classify(args) elif args.mode == 'dataset': dataset.build_dataset(args) elif args.mode == 'run': run_thread = run.MyThread(0, args) network_thread = run.MyThread(1, args) run_thread.start() network_thread.start() while network_thread.is_alive(): continue elif args.mode == 'normalize': start_time = time.clock() if args.split_dir != 'none': splits = dataset.split_images(args, start_time) log( args, '{:.5f}'.format(time.clock() - start_time) + 's ' + 'Images have been split ') if args.normalize: dataset.normalize(args, start_time) if args.split_dir != 'none': dataset.normalize(args, start_time, dirs=splits) else: log(args, 'Please select a mode using the tag --mode, use --help for help.', True)
def main(argv): del argv # Unused. if FLAGS.eager_run: tf.config.experimental_run_functions_eagerly(True) model_type = GraphTransformer data_spec = json.load(open(FLAGS.data_spec)) if FLAGS.predict: src_vocab = Vocabulary.load(data_spec["source_vocab"]) tgt_vocab = Vocabulary.load(data_spec["target_vocab"]) eval_set = build_dataset( os.path.join(FLAGS.predict), max_num_src_tokens=data_spec["max_num_src_tokens"], max_num_tgt_tokens=data_spec["max_num_tgt_tokens"], src_vocab=src_vocab, tgt_vocab=tgt_vocab, is_train=False, predict_edge=True, multiple=True) eval_set = eval_set.batch(FLAGS.batch_size, drop_remainder=False) predict(model_type, eval_set, src_vocab, tgt_vocab, FLAGS.save_model_path, json.load(open(FLAGS.ref_derivs)), FLAGS.predict_output) else: hparams = { "batch_sz": FLAGS.batch_size, "d_model": FLAGS.model_dim, "biaffine": FLAGS.biaffine, "num_src_tokens": data_spec["max_num_src_tokens"], "num_tgt_tokens": data_spec["max_num_tgt_tokens"], "dropout": FLAGS.dropout, "predict_edge": True } src_vocab = Vocabulary.load(data_spec["source_vocab"]) tgt_vocab = Vocabulary.load(data_spec["target_vocab"]) train_set = build_dataset(data_spec["train_set"], max_num_src_tokens=hparams["num_src_tokens"], max_num_tgt_tokens=hparams["num_tgt_tokens"], src_vocab=src_vocab, tgt_vocab=tgt_vocab, is_train=True, predict_edge=True, multiple=True) train_set = train_set.batch(FLAGS.batch_size, drop_remainder=True) dev_set = build_dataset(data_spec["dev_set"], max_num_src_tokens=hparams["num_src_tokens"], max_num_tgt_tokens=hparams["num_tgt_tokens"], src_vocab=src_vocab, tgt_vocab=tgt_vocab, is_train=False, predict_edge=True, multiple=True) dev_set = dev_set.batch(FLAGS.batch_size, drop_remainder=False) train(model_type, FLAGS.epochs, train_set, dev_set, src_vocab, tgt_vocab, hparams, FLAGS.save_model_path)
def main(args): if flags.FLAGS.prepare: directories = list_directories(flags.FLAGS.prepare) dataset = os.path.join(flags.FLAGS.prepare, 'combined') if not os.path.exists(dataset): os.makedirs(dataset) build_dataset(directories, dataset) elif flags.FLAGS.test: test(flags.FLAGS.dataset) else: model_fn = get_model_fn(flags.FLAGS.model) train(flags.FLAGS.dataset, model_fn)
def build_astro_sets(batch_size): files = 'data/train1, data/train2, data/train3, data/train4, data/train5, data/train6, data/train7, data/train8' train = build_dataset(files, batch_size, reverse_time_series_prob=0.5, shuffle_filenames=True) val = build_dataset('data/val', batch_size, reverse_time_series_prob=0.5, shuffle_filenames=True) test = build_dataset('data/test', 1) return train, val, test
def train_model(option): train_loader = DataLoader(dataset=build_dataset('./data/train_x.pkl', './data/train_y.pkl'), batch_size=option.batch_size, shuffle=True) test_loader = DataLoader(dataset=build_dataset('./data/test_x.pkl', './data/test_y.pkl'), batch_size=option.batch_size, shuffle=True) model = biLSTM_CRF(option.embedding_size, option.hidden_size, option.dict_number, option.num_labels) optimizer = torch.optim.Adam(model.parameters(), lr=option.lr) if option.use_gpu: model.cuda() if option.pre_trained: model.load_state_dict(torch.load(option.pre_trained)) for epoch in range(option.epochs): print(epoch) # 创建评价 train_eva = evaluation(option.label_dict) test_eva = evaluation(option.label_dict) # 训练 model.train() for step, (batch_x, batch_y, batch_masks) in enumerate(train_loader): optimizer.zero_grad() if option.use_gpu: batch_x = batch_x.cuda() batch_y = batch_y.cuda() masks = masks.cuda() y_pred, loss = model(batch_x, batch_y, batch_masks) train_eva.add(y_pred, batch_y) loss.backward() optimizer.step() # 通过测试集验证 model.eval() for step, (batch_x, batch_y) in enumerate(test_loader): if option.use_gpu: batch_x = batch_x.cuda() batch_y = batch_y.cuda() masks = masks.cuda() y_pred, loss = model(batch_x, batch_y, batch_masks) test_eva.add(y_pred, batch_y) print("train:") print(train_eva.evaluate()) print("test:") print(test_eva.evaluate())
def build(hidden_size, batch_size, max_len, cuda): bidirectional = False model_name = 'bert' x = import_module('models.' + model_name) config = x.Config(batch_size) train_data = build_dataset(config) train_dataloader = build_iterator(train_data, config) val_data, test_data = build_dataset_eval(config) val_dataloader = build_iterator_eval(val_data, config) test_dataloader = build_iterator_eval(test_data, config) encoder = x.Model(config).to(config.device) decoder = DecoderRNN(len(config.tokenizer.vocab), max_len, hidden_size * 2 if bidirectional else hidden_size, dropout_p=0.2, use_attention=True, bidirectional=bidirectional, eos_id=config.tokenizer.convert_tokens_to_ids([SEP ])[0], sos_id=config.tokenizer.convert_tokens_to_ids([CLS ])[0]) seq2seq = Seq2seq(encoder, decoder) if cuda: seq2seq.cuda() optimizer = torch.optim.Adam(lr=1e-3, params=seq2seq.parameters()) Tensor = torch.cuda.LongTensor if cuda else torch.LongTensor if cuda: seq2seq.cuda() Tensor = torch.cuda.LongTensor if cuda else torch.LongTensor loss_fun = torch.nn.NLLLoss(reduce=False) return seq2seq, optimizer, Tensor, train_dataloader, val_dataloader, test_dataloader, loss_fun, config
def __init__(self): self._flags = tf.flags.FLAGS tf.logging.warn('tensorflow flags:') for key, value in sorted(self._flags.flag_values_dict().items()): tf.logging.warn('{:25}: {}'.format(key, value)) self.dataset_args = { 'duration_thresholds': self._flags.duration_thresholds, 'weight_key': self._flags.weight_key, 'debug_labels': self._flags.debug_labels, 'use_sequence_weight': self._flags.use_sequence_weight, 'label_dtype': self._flags.label_dtype, 'mode': self._flags.mode, 'label_num': self._flags.label_num, 'multi_task_config': self._flags.multi_task_config, 'negative_rate': self._flags.negative_rate } self.Dataset = build_dataset(self._flags.dataset) self._dataset = self.Dataset(self._flags.compression_type, self._flags.label_key, self._flags.schema, self._flags.valid_path, self._flags.train_path, batch_size=self._flags.batch_size, prebatch=self._flags.prebatch, epochs=self._flags.epochs, seq_train=self._flags.seq_train, **self.dataset_args) self._Network = network_factory(self._flags.network) self._features = self._serving_features() self._build_estimator() self._assets = self._serving_warm_up() self._output_cols = list()
def input_fn(params): ds = build_dataset(params['data_loader'], batch_size=params['batch_size'], shuffle=256).prefetch(1) features, labels = ds.make_one_shot_iterator().get_next() return features, labels
def _build_dataset(self): dataset = build_dataset(self._flags.dataset) if dataset != WeightedDataset: raise ValueError( 'Weighted LR model should be run with WeightedDataset, find {}' .format(dataset.__name__)) return dataset
def get(name, args=None): if name.startswith("physics"): physics_map = { "physics_human": "data/human_ann.json", "physics": "data/ann.json" } return dict( factory="COCODataset", args=dict(root="", ann_file=physics_map[name]), ) if name.startswith("annotated_physics"): physics_map = { "annotated_physics_human": "data/annotated_human_ann.json", "annotated_physics_train": "data/annotated_ann.json", "annotated_physics_val": "data/annotated_ann.json" } return build_dataset( EasyDict( NAME="OBJECT_PROPOSAL", OBJECT_DERENDER_FILE=physics_map[name], SPLIT="TRAIN" if "train" in name else "VAL" if "val" in name else "TEST", ROOT=DatasetCatalog.TRAIN_ROOT if "human" not in name else DatasetCatalog.HUMAN_ROOT, ), args)
def main(_): flags_dict = FLAGS.flag_values_dict() train_mode = flags_dict['train_mode'] print(train_mode) if train_mode is not TrainMode.FL: train_dataset = build_dataset(config.TRAIN_DATA_PATH, positive_only=True).shuffle(3200).batch( config.BATCH_SIZE) test_dataset = build_dataset(config.TEST_DATA_PATH, positive_only=True).shuffle(3200).batch( config.BATCH_SIZE) else: train_dataset = build_dataset(config.FL_TRAIN_DATA_PATH, positive_only=False).shuffle(3200).batch( config.BATCH_SIZE) test_dataset = build_dataset(config.FL_TEST_DATA_PATH, positive_only=False).shuffle(3200).batch( config.BATCH_SIZE) if train_mode in [TrainMode.GAN, TrainMode.ALL]: print( "************************* Training GAN ***************************" ) gan_train(train_dataset, config.EPOCHS) elif train_mode in [TrainMode.AE, TrainMode.AAE, TrainMode.ALL]: print( "************************* Training AE ***************************" ) ae_train(train_dataset, test_dataset, config.EPOCHS, train_mode) elif train_mode == TrainMode.VAEGAN: print( "************************* Training VAE-GAN ***************************" ) # vaegan_train(train_dataset, test_dataset, pretrained_path='/data2/huangps/data_freakie/gx/code/elevator_GAN_FH/vaegan_training_checkpoints/ckpt_epoch_60-60') vaegan_train(train_dataset, test_dataset) elif train_mode == TrainMode.FL: print( "************************* Training VAE-GAN ***************************" ) classifier_train(train_dataset, test_dataset)
def main(config): run_logger = None if config['azure_ml']: run_logger = init_azure_logging(config) if config['tall_kernel']: model = tall_kernel(config) else: model = cnn(config) with tf.device('/cpu:0'): # put data pipeline on CPU ds_train = build_dataset(config, 'train') ds_val = build_dataset(config, 'val') loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True) optimizer = get_optimizer(config) model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) add_logpaths(config) save_state(config) callbacks = build_callbacks(config, run_logger) model.fit(x=ds_train, validation_data=ds_val, epochs=config['epochs'], callbacks=callbacks, validation_steps=90, verbose=config['verbose'])
def main(config): genc = build_genc() writer_config = build_writer_config(config) reader_config = build_dataset_reader_config(config) for mode in ['train', 'val', 'test']: writer = MyTFRWriter(writer_config, mode) ds = dataset.build_dataset(reader_config, mode) for batch in ds: x, y = batch x = genc(x) # shape = (batch_sz, 63, dim_z) for idx in range(x.shape[0]): example = serialized_example(x[idx, ...], y[idx]) writer.write(example) writer.close()
def train_input_fn(params): levels = build_levels() ds = dataset.build_dataset(params['data_loader'], levels=levels, scale=params['scale'], shuffle=4096, augment=True) ds = ds.map(lambda input: { **input, 'image': preprocess_image(input['image']) }) return ds.prefetch(1)
def get_loader(args): train_transform = transforms.Compose([ transforms.RandomResizedCrop(args.cropsize, scale=(args.crop, 1.)), transforms.RandomApply([transforms.ColorJitter(0.4, 0.4, 0.4, 0.1)], p=0.8), transforms.RandomGrayscale(p=0.2), transforms.RandomApply([GaussianBlur([.1, 2.])], p=0.5), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) train_dataset = build_dataset(args.dataset, args.data_dir, transform=train_transform, split = args.datasplit) train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True, sampler=train_sampler, drop_last=True) return train_loader
def train(): parser = argparse.ArgumentParser() train_dataset = '../data/corpus.small' vocab_dataset = '../data/vocab.small' output = '../output/bert.model' model = 'bert' parser.add_argument('--model', type=str, required=False, default=model) parser.add_argument("-c", "--train_dataset", required=False, type=str, default=train_dataset, help="train dataset for train bert") parser.add_argument("-t", "--test_dataset", type=str, default=None, help="test set for evaluate train set") parser.add_argument("-v", "--vocab_path", required=False, type=str, default=vocab_dataset, help="built vocab model path with bert-vocab") parser.add_argument("-o", "--output_path", required=False, type=str, default=output, help="exoutput/bert.model") parser.add_argument("-b", "--batch_size", type=int, default=64, help="number of batch_size") parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") parser.add_argument("--require_improvement", type=int, default=1000, help="patience of early stopping") parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") parser.add_argument("--log_freq", type=int, default=10, help="printing loss every n iter: setting n") parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") parser.add_argument("--cuda_devices", type=int, nargs='+', default=0, help="CUDA device ids") parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") args = parser.parse_args() model_name = args.model module = import_module('models.' + model_name) config = module.ModelConfig(args) train_data, dev_data, test_data = build_dataset(config) train_iter = build_iterator(train_data, config) dev_iter = build_iterator(dev_data, config) test_iter = build_iterator(test_data, config) model = module.Model(config).to(config.device) model_train(config, model, train_iter, dev_iter, test_iter) test(config, model, test_iter)
def evaluate(model, word2id, id2word, step, path_submission): # Create Dataset ds_test = build_dataset(PATH_TEST, word2id) ds_test = ds_test.batch(BATCH_SIZE) # ds_test = ds_test.take(2) # uncomment for demo purposes # Run Validation accuracy, perp = validate(model=model, dataset=ds_test, id2word=id2word, step=step) # Write Submission File perp = tf.concat(perp, axis=0) print(f"Writing Submission: {path_submission}") pd.DataFrame(perp.numpy()).to_csv(path_submission, sep=' ', header=False, index=False, float_format='%.3f')
def do_training(args): trainloader, testloader = build_dataset( args.dataset, dataroot=args.dataroot, batch_size=args.batch_size, eval_batch_size=args.eval_batch_size, num_workers=2) model = build_model(args.arch, num_classes=num_classes(args.dataset)) model = torch.nn.DataParallel(model).cuda() # Calculate total number of model parameters num_params = sum(p.numel() for p in model.parameters()) track.metric(iteration=0, num_params=num_params) optimizer = build_optimizer('SGD', params=model.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() best_acc = 0.0 for epoch in range(args.epochs): track.debug("Starting epoch %d" % epoch) args.lr = adjust_learning_rate(epoch, optimizer, args.lr, args.schedule, args.gamma) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch) test_loss, test_acc = test(testloader, model, criterion, epoch) track.debug( 'Finished epoch %d... | train loss %.3f | train acc %.3f | test loss %.3f | test acc %.3f' % (epoch, train_loss, train_acc, test_loss, test_acc)) # Save model model_fname = os.path.join(track.trial_dir(), "model{}.ckpt".format(epoch)) torch.save(model, model_fname) if test_acc > best_acc: best_acc = test_acc best_fname = os.path.join(track.trial_dir(), "best.ckpt") track.debug("New best score! Saving model") torch.save(model, best_fname)
def get_loader(args): val_transform_list = [] if args.resize > 0: val_transform_list.append(transforms.Resize(args.resize)) val_transform_list.append(transforms.CenterCrop(args.cropsize)) val_transform_list.append(transforms.ToTensor()) val_transform_list.append(transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])) val_transform = transforms.Compose(val_transform_list) clipdataset = build_dataset( dataset=args.dataset, root=args.data_dir, split=args.datasplit, transform=val_transform, sample_num=args.datasamplenum, ) dataloader = DataLoader(clipdataset, batch_size=1, num_workers=8, sampler=DistributedSampler(clipdataset, shuffle=False), shuffle=False, pin_memory=True, drop_last=False) return dataloader, len(clipdataset)
def main(argv): if FLAGS.eager_run: tf.config.experimental_run_functions_eagerly(True) model_type = Transformer data_spec = json.load(open(FLAGS.data_spec)) src_vocab = Vocabulary.load(data_spec["source_vocab"]) tgt_vocab = Vocabulary.load(data_spec["target_vocab"]) eval_set = build_dataset(os.path.join(FLAGS.predict), max_num_src_tokens=data_spec["max_num_src_tokens"], max_num_tgt_tokens=data_spec["max_num_tgt_tokens"], src_vocab=src_vocab, tgt_vocab=tgt_vocab, is_train=False) eval_set = eval_set.batch(FLAGS.batch_size, drop_remainder=False) get_top_beam_graphs(model_type, eval_set, src_vocab, tgt_vocab, FLAGS.save_model_path, FLAGS.output_path, graph_size=FLAGS.graph_size)
) # If the restored model is ready for inference, save it and quit training. if args.export_only: if latest_checkpoint is None: print("Warning: Model not restored from any checkpoint.") print("Saving model to {} ...".format(export_dir)) model.save(export_dir) print("Model saved at: {}".format(export_dir)) quit() # Construct a dataset for evaluation. dataset_test = build_dataset(test_files_dir, "test", number_marks=number_marks, image_shape=input_shape, training=False, batch_size=args.batch_size, shuffle=False, prefetch=tf.data.experimental.AUTOTUNE) # If only evaluation is required. if args.eval_only: model.evaluate(dataset_test) quit() # Finally, it's time to train the model. # Compile the model and print the model summary. model.compile(optimizer=keras.optimizers.Adam(0.0001), loss=keras.losses.MeanSquaredError(), metrics=[keras.metrics.MeanSquaredError()])
import numpy as np from six.moves import urllib import six.moves.urllib from six.moves import xrange # pylint: disable=redefined-builtin import tensorflow as tf # Read the data into a list of strings. words = ds.get_words() print('Data size', len(words)) vocabulary_size = 50000 data, count, dictionary, reverse_dictionary = ds.build_dataset(words, vocabulary_size) del words # Hint to reduce memory. print('Most common words (+UNK)', count[:5]) print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]]) data_index = 0 # Step 3: Function to generate a training batch for the skip-gram model. def generate_batch(batch_size, num_skips, skip_window): global data_index assert batch_size % num_skips == 0 assert num_skips <= 2 * skip_window batch = np.ndarray(shape=(batch_size), dtype=np.int32) labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) span = 2 * skip_window + 1 # [ skip_window target skip_window ]
def train_single_gpu(config, args): #setup logger str_t = datetime.now().strftime('%Y-%m-%d-%H-%M-%S') train_dir = f'./res/{str_t}' config['train']['train_dir'] = train_dir os.makedirs(train_dir) os.makedirs(train_dir + '/result') os.makedirs(train_dir + '/model') fh = logging.FileHandler(f'{train_dir}/train.log') fh.setFormatter(logging_formatter) logger.addHandler(fh) logger.info(json.dumps(config, indent=4)) #train and validate files batch_size = config['train']['batch_size'] input_config = config['input'] train_files = glob.glob(input_config['train']) validate_files = glob.glob(input_config['validate']) test_files = glob.glob(input_config['test']) train_dataset = build_dataset(train_files, batch_size) validate_dataset = build_dataset(validate_files, batch_size) test_dataset = build_dataset(test_files, batch_size) #metric metric_train_loss = tf.keras.metrics.Mean(name='train_loss') #model model = ModelAttention(config['model']) if args.init_model is not None: x = tf.ones([batch_size, window_size, input_feature_dim], dtype=tf.float32) ref_aa = tf.ones((batch_size, ), dtype=tf.int32) alt_aa = tf.ones((batch_size, ), dtype=tf.int32) mask = tf.zeros((batch_size, window_size), dtype=tf.float32) model((ref_aa, alt_aa, x), training=False, mask=mask) model.load_weights(args.init_model) #set other layers untrainable for layer in model.layers: if layer.name == 'gru_cell': layer.trainable = True else: layer.trainable = True print(layer.name, layer.trainable) #double check print('trainable variables') for var in model.trainable_variables: print(var.name) #learning rate init_learning_rate = config['train']['learning_rate'] end_learning_rate = config['train']['end_learning_rate'] warmup_steps, decay_steps = config['train']['warmup_steps'], config[ 'train']['decay_steps'] learning_rate = LearningRate(init_learning_rate, end_learning_rate=end_learning_rate, warmup_steps=warmup_steps, decay_steps=decay_steps) #training algorithm opt = config['train'].get('opt', 'adam') if opt == 'adam': optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate) else: raise NotImplementedError(f"opt {opt} not NotImplementedError") def _save_res(var_id, target, pred, name, epoch): with open(f'{train_dir}/result/epoch_{epoch}_{name}.score', 'w') as f: f.write('var\ttarget\tScore\n') for a, c, d in zip(var_id, target, pred): f.write('{}\t{:d}\t{:f}\n'.format(a.numpy().decode('utf-8'), int(c), d)) return True @tf.function(input_signature=[train_dataset.element_spec]) def test_step(sample): var, ref_aa, alt_aa, feature, label, padding_mask = sample logit = model((ref_aa, alt_aa, feature), False, padding_mask) loss = compute_loss(label, logit) pred = model.predict_from_logit(logit) return var, label, pred, loss def test(test_dataset, data_name, epoch, auc=False, pvalue=False, corr=False): #metrics metric_test = TestMetric() metric_test_loss = tf.keras.metrics.Mean(name='test_loss') metric_test.reset_states() metric_test_loss.reset_states() all_pred, all_label, all_var = [], [], [] for step, sample in enumerate(test_dataset): var, label, pred, loss = test_step(sample) metric_test.update_state(label, pred) metric_test_loss.update_state(loss) all_pred.extend(list(pred)) all_label.extend(list(label)) all_var.extend(list(var)) all_var = np.array(all_var) all_label = np.array(all_label) all_pred = np.array(all_pred) _save_res(all_var, all_label, all_pred, data_name, epoch) if auc: logger.info( f'{data_name} pos= {metric_test.result_pos()} neg= {metric_test.result_neg()} loss= {metric_test_loss.result()} auPR= {metric_test.result_auPR()} auROC= {metric_test.result_auROC()} max= {metric_test.result_max()}' ) if pvalue: logger.info( f'{data_name} pos= {metric_test.result_pos()} neg= {metric_test.result_neg()} loss= {metric_test_loss.result()} pvalue= {metric_test.result_pvalue()}' ) if corr: corr, pvalue = metric_test.result_corr() logger.info( f'{data_name} pos= {metric_test.result_total()} corr= {corr} pvalue= {pvalue} max= {metric_test.result_max()}' ) return metric_test.result_auROC() @tf.function(input_signature=[train_dataset.element_spec]) def train_step(sample): var, ref_aa, alt_aa, feature, label, padding_mask = sample with tf.GradientTape() as tape: logit = model((ref_aa, alt_aa, feature), True, padding_mask) loss = compute_loss(label, logit) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) metric_train_loss.update_state(loss) #if optimizer.iterations % 512 == 0: # _update_gradient_norm_summary(model.trainable_variables, gradients) return loss print('init model') #test(test_dataset, 'test', 0, pvalue=False, auc=True, corr=False) EPOCHS = 512 watch_auROC = -1.0 watch_epoch = -1 patience_epochs = 5 for epoch in range(EPOCHS): start = time.time() for step, samples in enumerate(train_dataset): loss = train_step(samples) #model summary if optimizer.iterations == 1: model.summary(print_fn=logger.info) logger.info(f'Epoch {epoch} Loss {metric_train_loss.result():.4f}') metric_train_loss.reset_states() model.save_weights(f'{train_dir}/model/epoch-{epoch}.h5') #validate and test validate_auROC = test(validate_dataset, 'validate', epoch, pvalue=False, auc=True, corr=False) if validate_auROC > watch_auROC: watch_auROC = validate_auROC watch_epoch = epoch if epoch - watch_epoch == patience_epochs: logger.info( f'best_epoch {watch_epoch} best_validate_auROC= {watch_auROC}') break
import dataset as ds ds.build_dataset('./data/images/numbers/train', 'jpg', './npz/service_numbers_train.npz') ds.plot_dataset('./data/service_numbers_train.npz') ds.build_dataset('./data/images/numbers/test', 'jpg', './npz/service_numbers_test.npz') ds.plot_dataset('./data/service_numbers_test.npz') ds.build_dataset('./data/images/letters/train', 'jpg', './npz/service_letters_train.npz') ds.plot_dataset('./data/service_letters_train.npz') ds.build_dataset('./data/images/letters/test', 'jpg', './npz/service_letters_test.npz') ds.plot_dataset('./data/service_letters_test.npz')
torch.manual_seed(seed) if use_cuda: os.environ['CUDA_VISIBLE_DEVICES'] = gpus torch.cuda.manual_seed(seed) model = Darknet(darknetcfg, learnetcfg) region_loss = model.loss model.load_weights(weightfile) model.print_network() ################################################### ### Meta-model parameters region_loss.seen = model.seen processed_batches = 0 if cfg.tuning else model.seen / batch_size trainlist = dataset.build_dataset(data_options) nsamples = len(trainlist) init_width = model.width init_height = model.height init_epoch = 0 if cfg.tuning else model.seen / nsamples max_epochs = max_batches * batch_size / nsamples + 1 max_epochs = int(math.ceil(cfg.max_epoch * 1. / cfg.repeat)) if cfg.tuning else max_epochs print(cfg.repeat, nsamples, max_batches, batch_size) print(num_workers) kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {} test_loader = torch.utils.data.DataLoader(dataset.listDataset( testlist, shape=(init_width, init_height), shuffle=False,
def _build_dataset(self): dataset = build_dataset(self._flags.dataset) if dataset not in (DurationDataset, MutiTaskDataset): raise ValueError('mmoe model should be run with DurationDataset or MutiTaskDataset, find {}' .format(dataset.__name__)) return dataset
def colorization_train(args, wandb_run): assert os.path.exists( args.draft_weights_path), f"{args.draft_weights_path} not exists" global logger, X logger = wandb_run os.makedirs(os.path.join(logger.dir, "image"), exist_ok=True) ##################### init values ########################## X = stats.truncnorm( (0 - args.mu) / args.sigma, (1 - args.mu) / args.sigma, loc=args.mu, scale=args.sigma, ) ########################## dataset ########################## train_transforms = ColorizationTransforms(args.image_size, args.draft_image_r, train=Mode.TRAIN) test_transforms = ColorizationTransforms(args.image_size, args.draft_image_r, train=Mode.TEST) train_dataset = build_dataset(os.path.join(args.root_dir, "train"), train_transforms, args.batch_size) test_dataset = build_dataset(os.path.join(args.root_dir, "test"), test_transforms, 8) test_batch = next(iter(test_dataset)) line, line_draft, hint, color = test_batch mask = opt.mask_gen(list(hint.shape), X, 0) hint = hint * mask test_batch = (line, line_draft, hint, color) ########################## model ########################## draft_image_size = args.image_size // args.draft_image_r draft_gen = Generator(args.g_dim) input_dict = {"line": line, "hint": hint} draft_gen(input_dict) draft_gen.load_weights(args.draft_weights_path) draft_gen.trainable = False gen = Generator(args.g_dim) ########################## optim ########################## total_step = len(train_dataset) * args.epochs gen_optim_lr_schedule = optim.schedules.ExponentialDecay( args.lr, decay_steps=int(total_step * args.decay_steps_rate), decay_rate=args.decay_rate, staircase=True, ) gen_optim = optim.Adam(gen_optim_lr_schedule, beta_1=args.beta_1, beta_2=args.beta_2) ######################## training ########################## for epoch in range(args.epochs): print(f"Epoch: {epoch} start") training_loop(train_dataset, test_batch, draft_gen, gen, gen_optim) gen.save_weights(os.path.join(logger.dir, "gen.ckpt.h5")) #################### artifacts loging ###################### artifacts_path = os.path.join(logger.dir, f"{args.mode}.h5") gen.save(artifacts_path) if args.upload_artifacts: gen_artifacts = wandb.Artifact(args.mode, type="model") gen_artifacts.add_file(artifacts_path, "weight") logger.log_artifact(gen_artifacts)
def main(): levels = build_levels() data_loader = Inferred('shapes', ['./tmp', 10, 500]) dataset = build_dataset(data_loader, levels=levels, scale=500, shuffle=4096, augment=True) input = dataset.make_one_shot_iterator().get_next() image = input['image'] classifications = input['detection']['classifications'] trainable_masks = input['trainable_masks'] regressions_postprocessed = utils.dict_starmap( lambda r, l: utils.regression_postprocess( r, tf.to_float(l.anchor_sizes / input['image_size'])), [input['detection']['regressions'], levels]) batch = [] for i in range(image.shape[0]): level_images = {} for k in levels: class_row = [] mask_row = [] boxes_row = [] for a in range(levels[k].anchor_sizes.shape[0]): image_with_classes = draw_classmap( image[i], classifications[k][i, :, :, a, :]) image_with_classes /= tf.reduce_max(image_with_classes) class_row.append(image_with_classes) image_with_mask = draw_mask(image[i], trainable_masks[k][i, :, :, a]) image_with_mask /= tf.reduce_max(image_with_mask) mask_row.append(image_with_mask) decoded = utils.boxes_decode( classifications[k][i, :, :, a, :], regressions_postprocessed[k][i, :, :, a, :]) image_with_boxes = draw_bounding_boxes(image[i], decoded, data_loader.class_names) image_with_boxes /= tf.reduce_max(image_with_boxes) boxes_row.append(image_with_boxes) class_row = tf.concat(class_row, 1) mask_row = tf.concat(mask_row, 1) boxes_row = tf.concat(boxes_row, 1) level_image = tf.concat([class_row, mask_row, boxes_row], 0) level_images[k] = level_image batch.append(level_images) with tf.Session() as sess: batch = sess.run(batch) for level_images in batch: for k in level_images: plt.figure(figsize=(16, 8)) plt.imshow(level_images[k]) plt.title(k) plt.show()
parser.add_argument("-t", "--table_path", type=str, required=True, help="The path of table file.") parser.add_argument("-w", "--image_width", type=int, default=100, help="Image width, this parameter will affect the output " "shape of the model, default is 100, so this model " "can only predict up to 24 characters.") parser.add_argument("-b", "--batch_size", type=int, default=256, help="Batch size.") parser.add_argument("-m", "--model", type=str, required=True, help="The saved model.") parser.add_argument("--channels", type=int, default=1, help="Image channels, " "0: Use the number of channels in the image, " "1: Grayscale image, " "3: RGB image") parser.add_argument("--ignore_case", action="store_true", help="Whether ignore case.(default false)") args = parser.parse_args() eval_ds, size, num_classes = build_dataset( args.annotation_paths, args.table_path, args.image_width, args.channels, args.ignore_case, batch_size=args.batch_size) print("Num of eval samples: {}".format(size)) model = keras.models.load_model(args.model, compile=False) model.summary() model.compile(loss=CTCLoss(), metrics=[WordAccuracy()]) model.evaluate(eval_ds)
def main(): args = arg_parser() seed_everything(args.seed) if cuda.is_available(): device = torch.device("cuda:0") else: device = torch.device("cpu") train_df = pd.read_csv(args.train_df_path) valid_df = pd.read_csv(args.valid_df_path) valid_df_sub = valid_df.sample( frac=1.0, random_state=42).reset_index(drop=True)[:40000] valid_df_sub1 = valid_df.sample( frac=1.0, random_state=52).reset_index(drop=True)[:40000] valid_df_sub2 = valid_df.sample( frac=1.0, random_state=62).reset_index(drop=True)[:40000] del valid_df gc.collect() if args.DEBUG: train_df = train_df[:1000] valid_df_sub = valid_df_sub[:1000] valid_df_sub1 = valid_df_sub1[:1000] valid_df_sub2 = valid_df_sub2[:1000] train_loader = build_dataset(args, train_df, is_train=True) batch_num = len(train_loader) valid_loader = build_dataset(args, valid_df_sub, is_train=False) valid_loader1 = build_dataset(args, valid_df_sub1, is_train=False) valid_loader2 = build_dataset(args, valid_df_sub2, is_train=False) model = build_model(args, device) if args.model == 'resnet50': save_path = os.path.join(args.PATH, 'weights', f'resnet50_best.pt') if args.model == 'resnext': save_path = os.path.join(args.PATH, 'weights', f'resnext_best.pt') elif args.model == 'xception': save_path = os.path.join(args.PATH, 'weights', f'xception_best.pt') else: NotImplementedError optimizer = build_optimizer(args, model) scheduler = build_scheduler(args, optimizer, batch_num) train_cfg = { 'train_loader': train_loader, 'valid_loader': valid_loader, 'valid_loader1': valid_loader1, 'valid_loader2': valid_loader2, 'model': model, 'criterion': nn.BCEWithLogitsLoss(), 'optimizer': optimizer, 'scheduler': scheduler, 'save_path': save_path, 'device': device } train_model(args, train_cfg)