def main(): display_config() dataset_root = get_full_path(args.scale, args.train_set) print('Contructing dataset...') dataset_factory = DatasetFactory() train_dataset = dataset_factory.create_dataset(args.model, dataset_root) model_factory = ModelFactory() model = model_factory.create_model(args.model) loss_fn = get_loss_fn(model.name) check_point = os.path.join('check_point', model.name, str(args.scale) + 'x') solver = Solver(model, check_point, loss_fn=loss_fn, batch_size=args.batch_size, num_epochs=args.num_epochs, learning_rate=args.learning_rate, fine_tune=args.fine_tune, verbose=args.verbose) print('Training...') solver.train(train_dataset)
def run_train(params: dict) -> Tuple[threading.Thread, threading.Thread]: """Train a network on a data generator. params -> dictionary. Required fields: * model_name * generator_name * dataset_dir * tile_size * clf_name * checkpoints_dir * summaries_dir Returns prefetch thread & model.fit thread""" assert 'model_name' in params assert 'generator_name' in params Model = ModelFactory.get_model(params['model_name']) Generator = GeneratorFactory.get_generator(params['generator_name']) model = Model(**params) feed = Generator(**params) pf = PreFetch(feed) t1 = threading.Thread(target=pf.fetch) t2 = threading.Thread(target=model.fit, args=(pf,)) t1.start() t2.start() return t1,t2
def main(): display_config() dataset_root = get_full_path(args.scale, args.test_set) print('Contructing dataset...') dataset_factory = DatasetFactory() train_dataset = dataset_factory.create_dataset(args.model, dataset_root) model_factory = ModelFactory() model = model_factory.create_model(args.model) check_point = os.path.join('check_point', model.name, str(args.scale) + 'x') solver = Solver(model, check_point) print('Testing...') stats, outputs = solver.test(train_dataset) export(args.scale, model.name, stats, outputs)
def test(logger, args): """ Test trained model on set-alone data; this should be done after all tunings """ raw_test = lg_utils.load_data_from_pickle( "{}s_test.p".format(args.task_type), args.data_size) data = [(p.get_x(), p.get_y()) for p in raw_test] model = ModelFactory().get_trained_model(logger, args) single_batches = model.make_padded_batch(data, 1) model.log_and_print("Test Evaluation") model.evaluate_core(single_batches, args)
def main(): display_config() print('Contructing dataset...') os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu train_dataset = VSR_Dataset(dir=args.train_set, trans=transforms.Compose([ RandomCrop(48, args.scale), DataAug(), ToTensor() ])) model_factory = ModelFactory() model = model_factory.create_model(args.model) model_parameters = filter(lambda p: p.requires_grad, model.parameters()) params = sum([np.prod(p.size()) for p in model_parameters]) print(1.0 * params / (1000 * 1000)) loss_fn = get_loss_fn(model.name) check_point = os.path.join(args.checkpoint, model.name, str(args.scale) + 'x') if not os.path.exists(check_point): os.makedirs(check_point) solver = Solver(model, check_point, model.name, loss_fn=loss_fn, batch_size=args.batch_size, num_epochs=args.num_epochs, learning_rate=args.learning_rate, fine_tune=args.fine_tune, verbose=args.verbose) print('Training...') val_dataset = VSR_Dataset(dir=args.test_set, trans=transforms.Compose([ToTensor()])) solver.train(train_dataset, val_dataset)
def __init__(self, args): self.dataset = Dataset(args) self.model = ModelFactory.newModel(args, self.dataset.usz, self.dataset.isz, self.dataset.fsz) print 'model', self.model self.epoch = args.epoch self.verbose = args.verbose self.adv = args.adv self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=0) self.sess.run(self.model.assign_image, feed_dict={self.model.init_image: self.dataset.emb_image}) self.weight_dir = args.weight_dir + '/' self.load()
class Test(unittest.TestCase): def setUp(self): if not hasattr(self, 'model'): self.model = ModelFactory(ModuleFactory(SPSConfig())).get_built() self.paths = 120 self.length = 730 def testgeneratepath(self): for thickness in range(1, 2): self.model.constants['SCREEN_THICKNESS'].value = thickness paths = self.paths length = self.length # 生成paths条时间长度为length的随机路径 for _ in range(paths): _, path = self.model.gen_random_path(length) logger.info('{0}, {1}'.format(thickness, path))
def test_on_fold( fold: int, data_of_fold: Dict[int, Tuple[List[Tensor], List[int]]], train_folds_of_fold: Dict[int, List[int]], model_factory: ModelFactory) -> Tuple[float, float, np.ndarray]: print(f"Testing fold {fold}") train_items, train_labels = train_data_for_fold(fold, data_of_fold, train_folds_of_fold) test_items, test_labels = data_of_fold[fold] model = model_factory.get() model.fit(train_items, train_labels) train_prediction = model.predict(train_items) test_prediction = model.predict(test_items) train_accuracy = accuracy_score(train_labels, train_prediction) test_accuracy = accuracy_score(test_labels, test_prediction) test_confusion = confusion_matrix(test_labels, test_prediction) return train_accuracy, test_accuracy, test_confusion
def train(logger, args): """ Training model """ # Load in data raw_train = lg_utils.load_data_from_pickle( "{}s_train.p".format(args.task_type), args.data_size) raw_cv = lg_utils.load_data_from_pickle("{}s_cv.p".format(args.task_type), args.data_size) print("Loading done") logger.info("Loading done") # Set up model if args.start_from_epoch >= 0: model = ModelFactory().get_trained_model(logger, args) else: # Set up encoders char_encoder = XEncoder(args) if args.task_type == "page": tag_encoder = YEncoder([config.INS_TAG, config.EOS_TAG]) else: tagset = sorted( list( set( itertools.chain.from_iterable( [r.orig_tags for r in raw_train])))) tag_encoder = YEncoder(tagset) model = ModelFactory().get_new_model(logger, args, char_encoder, tag_encoder) # Training # Step 1. Data preparation training_data = [(p.get_x(), p.get_y()) for p in raw_train] cv_data = [(p.get_x(), p.get_y()) for p in raw_cv] # Step 2. Model training if args.need_train: model.train_model(training_data, cv_data, args) else: model = ModelFactory().get_trained_model(logger, args)
def test(model_path, dataset_path): log_path = os.path.join(model_path, 'test_log') config = ConfigReader(os.path.join(model_path, 'config.json')).read() dataset_extractor = DatasetExtractor(dataset_path, config) dataset_test, test_size = dataset_extractor.extract() test_iterator = dataset_test.make_one_shot_iterator() dataset_handle = tf.placeholder(tf.string, shape=[]) feedable_iterator = tf.data.Iterator.from_string_handle(dataset_handle, dataset_test.output_types, dataset_test.output_shapes, dataset_test.output_classes) signal, label, signal_len, _ = feedable_iterator.get_next() label = tf.cast(label, dtype=tf.int32) model = ModelFactory.get(config.model_name, signal, config) optimizer = OptimizerFactory.get(config.optimizer, model.logits, label, signal_len) decoder = DecoderFactory.get(config.decoder, model.logits, signal_len) distance_op = tf.reduce_mean(tf.edit_distance(tf.cast(decoder.decoded, dtype=tf.int32), label)) saver = tf.train.Saver() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) saver.restore(sess, os.path.join(model_path, "model.ckpt")) tf.saved_model.simple_save(sess, os.path.join(model_path, 'saved_model'), inputs={"signal": signal, "lengths": signal_len}, outputs={"logits": model.logits})
base_path = os.path.abspath("UrbanSound8K") audio_path = os.path.normpath(f"{base_path}/audio") spectrogram_path = os.path.normpath(f"{base_path}/spectrogram") if not os.path.exists(spectrogram_path): resample_rate = 20050 mel_filters = 64 seconds = 3 preprocess(audio_path, spectrogram_path, all_folds, resample_rate, mel_filters, seconds) device = torch.cuda.current_device() if torch.cuda.is_available( ) else torch.device('cpu') classes = 10 batch_size = 8 epochs = 6 lr = 0.001 momentum = 0.9 model_factory = ModelFactory(device, pretrained_resnet18(classes), shuffle_batch_loader(batch_size), sgd_with_momentum(lr, momentum), epochs) train_accuracy, test_accuracy, test_confusion = test_model( model_factory, spectrogram_path, all_folds) print(f"Train Accuracy: {train_accuracy}") print(f"Test Accuracy: {test_accuracy}") print(f"Test Confusion: {test_confusion}")
def train(config_path, train_dataset_path, val_dataset_path, output_path, early_stop=False): keep_training = True if not os.path.exists(output_path): os.makedirs(output_path) copyfile(config_path, os.path.join(output_path, 'config.json')) log_path = os.path.join(output_path, 'log') config = ConfigReader(config_path).read() train_dataset_extractor = DatasetExtractor(train_dataset_path, config) val_dataset_extractor = DatasetExtractor(val_dataset_path, config) dataset_train, train_size = train_dataset_extractor.extract() train_iterator = dataset_train.make_one_shot_iterator() dataset_val, val_size = val_dataset_extractor.extract() dataset_test = dataset_val.take(300) dataset_val = dataset_val.take(75) val_iterator = dataset_val.make_initializable_iterator() test_iterator = dataset_test.make_one_shot_iterator() dataset_handle = tf.placeholder(tf.string, shape=[]) feedable_iterator = tf.data.Iterator.from_string_handle( dataset_handle, dataset_train.output_types, dataset_train.output_shapes, dataset_train.output_classes) signal, label, signal_len, _ = feedable_iterator.get_next() label = tf.cast(label, dtype=tf.int32) model = ModelFactory.get(config.model_name, signal, config) optimizer = OptimizerFactory.get(config.optimizer, model.logits, label, signal_len) decoder = DecoderFactory.get(config.decoder, model.logits, signal_len) distance_op = tf.reduce_mean( tf.edit_distance(tf.cast(decoder.decoded, dtype=tf.int32), label)) saver = tf.train.Saver() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) if config.debug: sess = tf_debug.LocalCLIDebugWrapperSession(sess) sess.run(tf.global_variables_initializer()) training_handle = sess.run(train_iterator.string_handle()) validation_handle = sess.run(val_iterator.string_handle()) test_handle = sess.run(test_iterator.string_handle()) epoch = 0 steps = 0 previous_print_length = 0 losses = [] prev_val_distance = None prev_val_loss = None while keep_training: try: loss_value, _ = sess.run( [optimizer.loss, optimizer.optimizer], feed_dict={dataset_handle: training_handle}) losses.append(loss_value) steps += config.batch_size if previous_print_length > 0: print('\b' * previous_print_length, end='', flush=True) message = f"Epoch: {epoch} Step: {steps} Step Loss:{loss_value} Epoch Loss: {np.mean(losses)}" log_to_file(log_path, message) previous_print_length = len(message) print(message, end='', flush=True) if steps >= train_size: saver.save(sess, os.path.join(output_path, f"model.ckpt")) if config.validate or early_stop: distances = [] val_losses = [] sess.run(val_iterator.initializer) while True: try: distance, val_loss = sess.run( [distance_op, optimizer.loss], feed_dict={dataset_handle: validation_handle}) distances.append(distance) val_losses.append(val_loss) except tf.errors.InvalidArgumentError as e: log_to_file(log_path, e.message) raise e except tf.errors.OutOfRangeError: break mean_distance = np.mean(distances) mean_val_loss = np.mean(val_losses) if prev_val_distance is not None and prev_val_loss is not None: if prev_val_distance < mean_distance and prev_val_loss < mean_val_loss: keep_training = False prev_val_loss = mean_val_loss prev_val_distance = mean_distance print(flush=True) log_message = f"Epoch: {epoch} Validation Loss: {mean_val_loss} Edit Distance: {mean_distance}" print(log_message, flush=True) log_to_file(log_path, log_message) epoch += 1 steps = 0 previous_print_length = 0 losses = [] except tf.errors.OutOfRangeError: break # End of dataset saver.save(sess, os.path.join(output_path, "model.ckpt")) test_distances = [] test_losses = [] while True: try: test_distance, test_loss = sess.run( [distance_op, optimizer.loss], feed_dict={dataset_handle: test_handle}) test_distances.append(test_distance) test_losses.append(test_loss) except tf.errors.OutOfRangeError: break mean_test_distance = np.mean(test_distances) mean_test_loss = np.mean(test_losses) print(flush=True) log_message = f"Test Loss: {mean_test_loss} Edit Distance: {mean_test_distance}" print(log_message, flush=True) log_to_file(log_path, log_message)
parser = argparse.ArgumentParser(description=description) parser.add_argument('-m', '--model', metavar='M', type=str, default='TDAN', help='network architecture.') parser.add_argument('-s', '--scale', metavar='S', type=int, default=4, help='interpolation scale. Default 4') parser.add_argument('-t', '--test-set', metavar='NAME', type=str, default='/home/cxu-serve/u1/ytian21/project/video_retoration/TDAN-VSR/data/Vid4', help='dataset for testing.') parser.add_argument('-mp', '--model-path', metavar='MP', type=str, default='model', help='model path.') parser.add_argument('-sp', '--save-path', metavar='SP', type=str, default='res', help='saving directory path.') args = parser.parse_args() model_factory = ModelFactory() model = model_factory.create_model(args.model) dir_LR = args.test_set lis = sorted(os.listdir(dir_LR)) model_path = os.path.join(args.model_path, 'model.pt') if not os.path.exists(model_path): raise Exception('Cannot find %s.' %model_path) model = torch.load(model_path) model.eval() path = args.save_path if not os.path.exists(path): os.makedirs(path) for i in range(len(lis)): print(lis[i]) LR = os.path.join(dir_LR, lis[i], 'LR_bicubic')
predictions: Optional[List[Dict[str, float]]] error: Optional[str] class TrainRequest(BaseModel): split: Optional[List[float]] model: ModelName save: bool class TrainResponse(BaseModel): message: str error: Optional[str] MF = ModelFactory() app = FastAPI() @app.get("/", response_model=ModelResponse) async def root() -> ModelResponse: return ModelResponse(error="This is a test endpoint.") @app.get("/predict", response_model=ModelResponse) async def explain_api() -> ModelResponse: return ModelResponse( error="Send a POST request to this endpoint with 'features' data.") @app.post("/predict")
from keras.callbacks import ModelCheckpoint, TensorBoard, ReduceLROnPlateau, CSVLogger from callback import MultipleClassAUROC, MultiGPUModelCheckpoint from model import ModelFactory import json output_dir = "C:\\Users\\yanqing.yqh\\code\\wly-chexnet-keras\\modeltrain\\output" train_dir = ( "C:\\Users\\yanqing.yqh\\code\\wly-chexnet-keras\\cats_and_dogs_small\\train" ) validation_dir = ( "C:\\Users\\yanqing.yqh\\code\\wly-chexnet-keras\\cats_and_dogs_small\\validation" ) output_weights_path = os.path.join(output_dir, "weight.h5") class_names = ["dog", "cat"] model_factory = ModelFactory() model = model_factory.get_model(class_names) print(model.summary()) print(len(model.layers)) train_datagen = ImageDataGenerator( samplewise_center=True, samplewise_std_normalization=True, horizontal_flip=True, vertical_flip=False, height_shift_range=0.05, width_shift_range=0.1, rotation_range=5, shear_range=0.1, fill_mode="reflect", zoom_range=0.15,
def produce(logger, args): """ Produce untagged data using model; this step is unsupervised """ # Step 1. using page_to_sent_model, parse pages to sentences pages_produce = lg_utils.load_data_from_pickle("pages_produce.p", args.data_size) # pages_produce = lg_utils.load_data_from_pickle("pages_cv.p", args.data_size) # Step 2. depending on whether user wants to use RegEx/model, process page splitting if args.regex: with open(os.path.join(config.REGEX_PATH, "surname.txt"), 'r', encoding="utf8") as f: surnames = f.readline().replace("\ufeff", '') tag_seq_list = [] for p in pages_produce: tags = [config.INS_TAG for c in p.txt] for m in re.finditer(r"{}(".format(config.PADDING_CHAR) \ + surnames \ + ')', p.txt): tags[m.start( 0 )] = config.EOS_TAG # no need to -1, instead drop '○' before name tags[-1] = config.EOS_TAG tag_seq_list.append([config.BEG_TAG] + tags + [config.END_TAG]) else: vars(args)["task_type"] = "page" page_model = ModelFactory().get_trained_model(logger, args) pages_data = [p.get_x() for p in pages_produce] tag_seq_list = page_model.evaluate_model(pages_data, args) # list of list of tag # Step 3. using trained record model, tag each sentence # Get model vars(args)["task_type"] = "record" record_model = ModelFactory().get_trained_model(logger, args) record_test_data = [] # list of list of str records = [] # list of Record for p, pl in zip( pages_produce, lg_utils.get_sent_len_for_pages(tag_seq_list, config.EOS_TAG)): rs = p.separate_sentence(pl) # get a list of Record instances records.extend(rs) record_test_data.extend([r.get_x() for r in rs]) # Use trained model to process tagged_sent = record_model.evaluate_model(record_test_data, args) for record, tag_list in zip(records, tagged_sent): record.set_tag(tag_list) # Step 4. Saving curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") if args.saver_type == "html": saver = HtmlSaver(records) filename = os.path.join(config.OUTPUT_PATH, "test_{}.txt".format(curr_time)) else: raise ValueError("Unsupported save type: " + args.saver_type) # saver = ExcelSaver(records) # filename = os.path.join(config.OUTPUT_PATH, "test_{}.xlsx".format(curr_time)) saver.save(filename, record_model.y_encoder.tag_dict.values())
def setUp(self): if not hasattr(self, 'model'): self.model = ModelFactory(ModuleFactory(SPSConfig())).get_built() self.paths = 120 self.length = 730
from model import ModelFactory import tensorflow as tf import sys if __name__ == "__main__": placeholder = tf.placeholder(tf.float32, [None, 300, 1]) model = ModelFactory.get(sys.argv[1], placeholder)