def build(settings={}): gpu_num = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) settings['dev'] = '/gpu:0' net_dict_lst = [] net_dict_lst.append(net.build_model(settings['net'], settings)) train_op = net_dict_lst[-1]['opt' + settings['loss']] if gpu_num > 1: tower_grads = [] with tf.device('/cpu:0'): opt = tf.train.AdamOptimizer(net_dict_lst[0]['lr']) with tf.device(settings['dev']): grads = opt.compute_gradients(net_dict_lst[0][settings['loss']]) tower_grads.append(grads) for i in range(1, gpu_num): settings['dev'] = '/gpu:%d' % i settings['reuse'] = True net_dict_lst.append(net.build_model(settings['net'], settings)) with tf.device(settings['dev']): grads = opt.compute_gradients( net_dict_lst[0][settings['loss']]) tower_grads.append(grads) with tf.device('/gpu:0'): avg_grads = average_gradients(tower_grads) with tf.device('/cpu:0'): train_op = opt.apply_gradients( avg_grads, global_step=net_dict_lst[0][settings['step']]) print(settings['net'] + ' built on %d gpus' % gpu_num) return net_dict_lst, train_op
def main(args): config.merge_from_list(args.opts) cfg, logger = default_setup(config, args) if args.debug: batches = int(cfg.SOLVER.IMS_PER_DEVICE * args.num_gpus) if cfg.SOLVER.IMS_PER_BATCH != batches: cfg.SOLVER.IMS_PER_BATCH = batches logger.warning( "SOLVER.IMS_PER_BATCH is changed to {}".format(batches)) valid_files = get_valid_files(args, cfg, logger) # * means all if need specific format then *.csv for current_file in valid_files: cfg.MODEL.WEIGHTS = current_file model = build_model(cfg) DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) if cfg.TEST.AUG.ENABLED: res = Trainer.test_with_TTA(cfg, model) else: res = Trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res)
def main(args): config.merge_from_list(args.opts) cfg, logger = default_setup(config, args) model = build_model(cfg) logger.info(f"Model structure: {model}") file_sys = os.statvfs(cfg.OUTPUT_DIR) free_space_Gb = (file_sys.f_bfree * file_sys.f_frsize) / 2**30 # We assume that a single dumped model is 700Mb eval_space_Gb = (cfg.SOLVER.LR_SCHEDULER.MAX_ITER // cfg.SOLVER.CHECKPOINT_PERIOD) * 700 / 2**10 if eval_space_Gb > free_space_Gb: logger.warning(f"{Fore.RED}Remaining space({free_space_Gb}GB) " f"is less than ({eval_space_Gb}GB){Style.RESET_ALL}") if args.eval_only: DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) res = Trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res) if cfg.TEST.AUG.ENABLED: res.update(Trainer.test_with_TTA(cfg, model)) return res """ If you'd like to do anything fancier than the standard training logic, consider writing your own training loop or subclassing the trainer. """ trainer = Trainer(cfg, model) trainer.resume_or_load(resume=args.resume) if cfg.TEST.AUG.ENABLED: trainer.register_hooks([ hooks.EvalHook(0, lambda: trainer.test_with_TTA(cfg, trainer.model)) ]) return trainer.train()
def debug_train(): os.environ["CUDA_VISIBLE_DEVICES"] = "2" traindir = "/data4T1/samhu/shapenet_split_complete/train" settings = {} settings['batch_size'] = 32 settings['height'] = 192 settings['width'] = 256 net_name = 'KPARAM_33_DUAL' net_dict = net.build_model(net_name, settings) train_fetcher = DataFetcher() train_fetcher.BATCH_SIZE = settings['batch_size'] train_fetcher.PTS_DIM = 3 train_fetcher.HEIGHT = settings['height'] train_fetcher.WIDTH = settings['width'] train_fetcher.Dir = util.listdir(traindir, ".h5") train_fetcher.shuffleDir() if 'rand' in net_dict.keys(): train_fetcher.randfunc = net_dict['rand'] config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True lrate = 3e-5 with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) try: train_fetcher.start() lastEpoch = 0 data_dict = train_fetcher.fetch() x2D = data_dict['x2D'] x3D = data_dict['x3D'] yGT = 0.8 * data_dict['x3D_final'] GT_PTS_NUM = int(yGT.shape[1]) print x3D.shape print yGT.shape yGT = yGT.reshape((-1, 3)) x3D = x3D.reshape((-1, 3)) feed = { net_dict['yGT']: yGT, net_dict['ix3D']: x3D, net_dict['ix2D']: x2D, net_dict['lr']: lrate } if 'eidx' in data_dict.keys(): i = 0 while 'eidx_%d' % i in net_dict.keys(): feed[net_dict['eidx_%d' % i]] = data_dict['eidx'][i] i += 1 if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()): feed[net_dict['fidx']] = data_dict['fidx'][-1] yout = sess.run(net_dict['ox3D'], feed_dict=feed) print yout.shape train_fetcher.shutdown() finally: train_fetcher.shutdown() return
def main(args): config.merge_from_list(args.opts) cfg, logger = default_setup(config, args) if args.debug: batches = int(cfg.SOLVER.IMS_PER_BATCH / 8 * args.num_gpus) if cfg.SOLVER.IMS_PER_BATCH != batches: cfg.SOLVER.IMS_PER_BATCH = batches logger.warning( "SOLVER.IMS_PER_BATCH is changed to {}".format(batches)) if "MODEL.WEIGHTS" in args.opts: if cfg.MODEL.WEIGHTS.endswith(".pth") and not PathManager.exists( cfg.MODEL.WEIGHTS): ckpt_name = cfg.MODEL.WEIGHTS.split("/")[-1] model_prefix = cfg.OUTPUT_DIR.split("cvpods_playground")[1][1:] remote_file_path = os.path.join(cfg.OSS.DUMP_PREFIX, model_prefix, ckpt_name) logger.warning( f"The specified ckpt file ({cfg.MODEL.WEIGHTS}) was not found locally," f" try to load the corresponding dump file on OSS ({remote_file_path})." ) cfg.MODEL.WEIGHTS = remote_file_path valid_files = [cfg.MODEL.WEIGHTS] else: list_of_files = glob.glob(os.path.join(cfg.OUTPUT_DIR, '*.pth')) assert list_of_files, "No checkpoint file found in {}.".format( cfg.OUTPUT_DIR) list_of_files.sort(key=os.path.getctime) latest_file = list_of_files[-1] if not args.end_iter: valid_files = [latest_file] else: files = [f for f in list_of_files if str(f) <= str(latest_file)] valid_files = [] for f in files: try: model_iter = int(re.split(r'(model_|\.pth)', f)[-3]) except Exception: logger.warning("remove {}".format(f)) continue if args.start_iter <= model_iter <= args.end_iter: valid_files.append(f) assert valid_files, "No .pth files satisfy your requirement" # * means all if need specific format then *.csv for current_file in valid_files: cfg.MODEL.WEIGHTS = current_file model = build_model(cfg) DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) res = Trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res) if cfg.TEST.AUG.ENABLED: res.update(Trainer.test_with_TTA(cfg, model))
def main(_): tf.logging.set_verbosity(tf.logging.INFO) tf.keras.backend.set_learning_phase(0) model = build_model() model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy()) graph_def = K.get_session().graph.as_graph_def() graph_def = graph_util.extract_sub_graph(graph_def, [FLAGS.output_nodes]) tf.train.write_graph(graph_def, FLAGS.save_dir, FLAGS.graph_filename, as_text=True) print("Finish export inference graph: {}".format(FLAGS.save_dir))
def main(): parser = argparse.ArgumentParser( description='Example of Keras Data Generator') parser.add_argument('--train_dir', default='data/cifar/train/') parser.add_argument('--test_dir', default='data/cifar/train/') parser.add_argument('--label_file', default='data/cifar/labels.txt') parser.add_argument('--batchsize', '-b', type=int, default=32, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=100, help='Number of sweeps over the dataset to train') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') args = parser.parse_args() print('batchsize: {}'.format(args.batchsize)) print('epoch: {}'.format(args.epoch)) # Datasets train_dir = pathlib.Path(args.train_dir) train_datagen = ImageDataGenerator() test_dir = pathlib.Path(args.test_dir) test_datagen = ImageDataGenerator() classes = list(pd.read_csv(args.label_file, header=-1)[0]) input_shape = (32, 32, 3) # Model model = build_model(input_shape, len(classes)) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.9), metrics=['accuracy']) model.summary() # Train model.fit_generator( generator=train_datagen.flow_from_directory(train_dir, classes), steps_per_epoch=int( np.ceil(len(list(train_dir.iterdir())) / args.batchsize)), epochs=args.epoch, verbose=1, validation_data=test_datagen.flow_from_directory(test_dir, classes), validation_steps=int( np.ceil(len(list(test_dir.iterdir())) / args.batchsize)))
def main(args): config.merge_from_list(args.opts) cfg, logger = default_setup(config, args) if args.debug: batches = int(cfg.SOLVER.IMS_PER_BATCH / 8 * args.num_gpus) if cfg.SOLVER.IMS_PER_BATCH != batches: cfg.SOLVER.IMS_PER_BATCH = batches logger.warning( "SOLVER.IMS_PER_BATCH is changed to {}".format(batches)) if "MODEL.WEIGHTS" in args.opts: valid_files = [cfg.MODEL.WEIGHTS] else: list_of_files = glob.glob(os.path.join(cfg.OUTPUT_DIR, '*.pth')) assert list_of_files, "no pth file found in {}".format(cfg.OUTPUT_DIR) list_of_files.sort(key=os.path.getctime) latest_file = list_of_files[-1] if not args.end_iter: valid_files = [latest_file] else: files = [f for f in list_of_files if str(f) <= str(latest_file)] valid_files = [] for f in files: try: model_iter = int(re.split(r'(model_|\.pth)', f)[-3]) except Exception: logger.warning("remove {}".format(f)) continue if args.start_iter <= model_iter <= args.end_iter: valid_files.append(f) assert valid_files, "No .pth files satisfy your requirement" # * means all if need specific format then *.csv for current_file in valid_files: cfg.MODEL.WEIGHTS = current_file model = build_model(cfg) DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) res = Trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res) if cfg.TEST.AUG.ENABLED: res.update(Trainer.test_with_TTA(cfg, model))
def main(args): config.merge_from_list(args.opts) cfg = setup(args) model = build_model(cfg) logger.info("Model:\n{}".format(model)) if args.eval_only: DefaultCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) return do_test(cfg, model) distributed = comm.get_world_size() > 1 if distributed: model = DistributedDataParallel(model, device_ids=[comm.get_local_rank()], broadcast_buffers=False) do_train(cfg, model) return do_test(cfg, model)
def main(_): batch_size = 120 epochs = FLAGS.epochs model = build_model() model.compile(optimizer=tf.train.AdamOptimizer(0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy']) if FLAGS.pretrained: model.load_weights(FLAGS.pretrained) # Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset. model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1) score = model.evaluate(x_test, y_test, verbose=1) model.save_weights(FLAGS.ckpt_path, save_format='tf')
def train_net(): os.environ["CUDA_VISIBLE_DEVICES"] = '0' #指定第一块GPU可用 config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.6 # 程序最多只能占用指定gpu50%的显存 config.gpu_options.allow_growth = True #程序按需申请内存 sess = tf.Session(config=config) # 设置session KTF.set_session(sess) output_length = fs * 10 input_length = fs * 10 n_classes = 2 input_length = 1280 for fold in range(1, 6, 1): print("*******************fold {}*****************".format(fold)) tr_data_path = "./validation/train/validation_{}_data.npy".format( fold) #validation tr_label_path = "./validation/train/validation_{}_target.npy".format( fold) val_data_path = "./validation/val/validation_{}_data.npy".format(fold) val_label_path = "./validation/val/validation_{}_target.npy".format( fold) #cv X_tr = np.load(tr_data_path) y_tr = np.load(tr_label_path).reshape(-1, 1280, 1) X_val = np.load(val_data_path) y_val = np.load(val_label_path).reshape(-1, 1280, 1) # callback_lists checkpoint = ModelCheckpoint(filepath="./model/"+ \ 'model_weights_mse_0.08_1280_{}_fold_{}_ys.h5' \ .format(time.strftime('%Y-%m-%d %X').split(" ")[0],fold), monitor= 'val_loss', mode='min', verbose=1, save_best_only='True', save_weights_only='True') csv_logger = CSVLogger( filename='./log/log{}_fold{}.csv'.format( \ time.strftime('%Y-%m-%d %X').split(" ")[0],fold), separator=',', append=True) earlystop = EarlyStopping( monitor='val_loss', min_delta=0, patience=5, verbose=1, mode="min", #baseline=None, #restore_best_weights=True, ) reducelr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1, min_lr=1e-5) callback_lists = [earlystop, checkpoint, reducelr] input_layer = Input((input_length, 1)) output_layer = build_model(input_layer=input_layer, block="resnext", start_neurons=16, DropoutRatio=0.5, filter_size=32, nClasses=2) model = Model(input_layer, output_layer) #print(model.summary()) model.compile( loss='mse', #'categorical_crossentropy', optimizer=RAdam(1e-4), #RAdam(lr_schedule(0)),# metrics=['accuracy', 'mse', 'mae']) #focal_loss #mape history = model.fit( X_tr, y_tr, epochs=400, #350 train batch_size=32, verbose=2, validation_data=(X_val, y_val), callbacks=callback_lists) #class_weight=class_weight
samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=0, width_shift_range=0.125, height_shift_range=0.125, horizontal_flip=True, vertical_flip=False, fill_mode='nearest') test_datagen = ImageDataGenerator(rescale=1. / 255) print "loading original inception model" model = net.build_model(nb_classes) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=["accuracy"]) # train the model on the new data for a few epochs print "training the newly added dense layers" net.save(model, model_file_prefix) # we chose to train the top 2 inception blocks, i.e. we will freeze # the first 172 layers and unfreeze the rest: for layer in model.layers[:172]: layer.trainable = False for layer in model.layers[172:]:
samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=0, width_shift_range=0.125, height_shift_range=0.125, horizontal_flip=True, vertical_flip=False, fill_mode='nearest', ) logger.execution_time(dataset_start, "Dataset gathering and formating", 0) #2# Original InceptionV3 model loading model = net.build_model(classes_count) model.compile( optimizer='rmsprop', loss='categorical_crossentropy', metrics=[metrics.categorical_accuracy, metrics.top_k_categorical_accuracy]) logger.log("The Original InceptionV3 model has been loaded", 0) #3# Train the model on the new data for a few epochs and save logger.log("Model first train, evaluation and save", 0) first_train_start = time.time() filepath = MODEL_FILE_FULL_PATH + "_0.h5" # Cross validation loop for i, (train_index, test_index) in enumerate(skf.split(data, y)): logger.log("Folds " + str(i), 2)
def train(): INPUT_DATA = 'D:/Project/classify/demo/dataset/age/mtcnn_hunhe_train/' test_DATA = 'D:/Project/classify/demo/dataset/age/mtcnn_hunhe_train/' batch_size = 32 # initial_learning_rate = 1e-4 train_datagen = ImageDataGenerator( # preprocessing_function=preprocess_input_new, # ((x/255)-0.5)*2 归一化到±1之间 rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.5, horizontal_flip=True, ) train_generator = train_datagen.flow_from_directory(directory=INPUT_DATA, target_size=(image_size, image_size), # Inception V3规定大小 batch_size=batch_size) print(train_generator.class_indices) val_generator = train_datagen.flow_from_directory(directory=test_DATA, target_size=(image_size, image_size), # Inception V3规定大小 batch_size=batch_size) model = net.build_model(8, (image_size, image_size, 3)) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[myacc]) # model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) # metrics: 评价函数, 与损失函数类似, 只不过评价函数的结果不会用于训练过程中, 可以传递已有的评价函数名称, 或者传递一个自定义的theano / tensorflow函数来使用, 自带的评价函数有: binary_accuracy( # y_true, y_pred), categorical_accuracy(y_true, y_pred), sparse_categorical_accuracy(y_true, # y_pred), top_k_categorical_accuracy(y_true, y_pred, k=5).自定义评价函数应该在编译的时候compile传递进去, # 该函数需要以(y_true, y_pred) 作为输入参数, 并返回一个张量作为输出结果. try: os.mkdir("./model/") except OSError: print(OSError) pass checkpoint = ModelCheckpoint(filepath='./model/weights.{epoch:02d}.hdf5', monitor='val_loss', verbose=1, save_weights_only=False, period=1) path = 'D:/Project/classify/demo/dataset/age/mtcnn_hunhe_test/' testmodel = TestModel(model, path, train_generator) # lr_scheduler = LearningRateScheduler(ir_Schedule) # optimizer = Adam(lr=initial_learning_rate, decay=1e-6) # model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=[acc]) patience = 50 reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(patience / 4), verbose=1) # 可以自己定义compile函数中传入的myacc,也可以用系统定义的 acc,lr,loss,val_loss,val_acc,来作为检测项目。 callbacks = [checkpoint, reduce_lr, testmodel] model.fit_generator(generator=train_generator, steps_per_epoch=int(train_generator.samples / batch_size), #参数steps_per_epoch是通过把训练图像的数量除以批次大小得出的。 epochs=50, # 例如,有100张图像且批次大小为50,则steps_per_epoch值为2。参数epoch决定网络中所有图像的训练次数。 validation_data=val_generator, validation_steps=1, class_weight='auto', callbacks=callbacks) model.save('./models/%s.h5' % model_name) del train_generator del val_generator del model gc.collect()
def testlayers(settings={}): if not os.path.exists(preddir): os.mkdir(preddir); net_model = None; config = None; if not os.environ["CUDA_VISIBLE_DEVICES"]: net_dict = net.build_model(net_name,settings); config = tf.ConfigProto(intra_op_parallelism_threads=4,device_count={'gpu':0}); else: net_dict = net.build_model(net_name,settings); config = tf.ConfigProto(); config.gpu_options.allow_growth = True; config.allow_soft_placement = True; test_fetcher = DataFetcher(); test_fetcher.BATCH_SIZE = settings['batch_size']; test_fetcher.PTS_DIM = 3; test_fetcher.HEIGHT = settings['height']; test_fetcher.WIDTH = settings['width']; test_fetcher.Dir = util.listdir(traindir,".h5"); test_fetcher.useMix = False; FetcherLst.append(test_fetcher); if 'rand' in net_dict.keys(): test_fetcher.randfunc=net_dict['rand']; with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()); saver = tf.train.Saver(); ckpt = tf.train.get_checkpoint_state('%s/'%dumpdir); if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path); else: print "failed to restore model"; return; layers = []; li = 0; while 'ox3D%02d'%li in net_dict.keys(): layers.append(net_dict['ox3D%02d'%li]); li += 1; cnt = 30; test_fetcher.Cnt = cnt; try: test_fetcher.start(); data_dict = test_fetcher.fetch(); x2D = data_dict['x2D']; x3D = data_dict['x3D']; yGT = data_dict['yGT']; tag = test_fetcher.fetchTag(); # yGTout = yGT.copy(); yGT = yGT.reshape((-1,3)); x3D = x3D.reshape((-1,3)); feed={ net_dict['yGT']:yGT, net_dict['ix3D']:x3D, net_dict['ix2D']:x2D }; if 'eidx' in data_dict.keys(): i = 0; while 'eidx_%d'%i in net_dict.keys(): feed[net_dict['eidx_%d'%i]] = data_dict['eidx'][i]; i += 1; if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()): feed[net_dict['fidx']] = data_dict['fidx'][-1]; ylayers = sess.run(layers,feed_dict=feed); fdir = preddir+os.sep+"pred_%s_%03d"%(tag,cnt); if not os.path.exists(fdir): os.mkdir(fdir); i = 0; for layer in layers: lname = layer.name.replace(':','_'); if i < len(data_dict['fidx']): fidx = data_dict['fidx'][i]; else: fidx = data_dict['fidx'][-1]; f_lst = []; for j in range(test_fetcher.BATCH_SIZE): f_lst.append(fidx); util.write_to_obj(fdir+os.sep+"obj%02d"%i,ylayers[i],faces=f_lst); i += 1; finally: test_fetcher.shutdown(); return;
def test(settings={}): if not os.path.exists(preddir): os.mkdir(preddir); net_model = None; config = None; if not os.environ["CUDA_VISIBLE_DEVICES"]: net_dict = net.build_model(net_name,settings); config = tf.ConfigProto(intra_op_parallelism_threads=4,device_count={'gpu':0}); else: net_dict = net.build_model(net_name,settings); config = tf.ConfigProto(); config.gpu_options.allow_growth = True; config.allow_soft_placement = True; test_fetcher = DataFetcher(); test_fetcher.BATCH_SIZE = settings['batch_size']; test_fetcher.PTS_DIM = 3; test_fetcher.HEIGHT = settings['height']; test_fetcher.WIDTH = settings['width']; test_fetcher.Dir = util.listdir(traindir,".h5"); test_fetcher.useMix = False; FetcherLst.append(test_fetcher); if 'rand' in net_dict.keys(): test_fetcher.randfunc=net_dict['rand']; with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()); saver = tf.train.Saver(); ckpt = tf.train.get_checkpoint_state('%s/'%dumpdir); if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path); else: print "failed to restore model"; return; stat = {}; try: test_fetcher.start(); for cnt in range(len(test_fetcher.Dir)): data_dict = test_fetcher.fetch(); x2D = data_dict['x2D']; x3D = data_dict['x3D']; yGT = data_dict['yGT']; tag = test_fetcher.fetchTag(); r2D = None; if 'r2D' in net_dict.keys(): r2D_dim = int(net_dict['r2D'].shape[1]); r2D = np.random.normal(loc=0.0,scale=1.0,size=[BATCH_SIZE,r2D_dim]); # rgb = None; f_lst = []; if 'x3D_final' in data_dict.keys(): rgb = util.sphere_to_YIQ( data_dict['x3D_final'] ); # yGTout = yGT.copy(); yGT = yGT.reshape((-1,3)); x3D = x3D.reshape((-1,3)); feed={ net_dict['yGT']:yGT, net_dict['ix3D']:x3D, net_dict['ix2D']:x2D }; if 'eidx' in data_dict.keys(): i = 0; while 'eidx_%d'%i in net_dict.keys(): feed[net_dict['eidx_%d'%i]] = data_dict['eidx'][i]; i += 1; if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()): feed[net_dict['fidx']] = data_dict['fidx'][-1]; yout=None; loss=None; yout,loss = sess.run([net_dict['ox3D'],net_dict['chmf']],feed_dict=feed); fdir = preddir+os.sep+"pred_%s_%03d_%f"%(tag,cnt,loss); # if not os.path.exists(fdir): os.mkdir(fdir); if 'fidx' in data_dict.keys(): for j in range(yout.shape[0]): f_lst.append(data_dict['fidx'][-1]); util.write_to_obj(fdir+os.sep+"obj",yout,rgb,f_lst); else: util.write_to_obj(fdir+os.sep+"obj",yout); util.write_to_obj(fdir+os.sep+"GTobj",yGTout); #generating dense result util.write_to_img(fdir,x2D); finally: test_fetcher.shutdown(); return;
# 計算グラフを作成しない with torch.no_grad(): # 順伝播を実行し、推論結果を出力 y = net(x) y = sigm(y) misstooth = [dlabels[i] for i, x in enumerate((y < 0.3).tolist()[0]) if x] ##################### # SSD shit ##################### torch.set_default_tensor_type('torch.cuda.FloatTensor') # ネットワークの定義 # 引数が'test'だと、推論結果に対してクラスDetectで後処理を実行 ssd_net = build_model('SSD', 33) # initialize SSD # GPUへの転送 net = ssd_net.to(device) # 学習済みモデルのロード net.load_weights('./weights/DentalPano_v0.6(CLAHE, Adam).pth') dataset = VOCDetection(root=VOC_ROOT, target_transform=args['target_trans'], transform=SSDAugmentation()) data_loader = data.DataLoader(dataset, args['batch_size'], num_workers=args['num_workers'], shuffle=True,
import torch.utils.data as data import matplotlib.patches as patches import matplotlib.pyplot as plt # TODO: YO HERE! args = FRCNN_train model = args['model'] dataset = VOCDetection(root=VOC_ROOT, target_transform=args['target_trans'], transform=FRCNNAugmentation()) # ネットワークの定義 net = build_model(model, voc_config['num_classes']) device = 'cuda' if torch.cuda.is_available() else 'cpu' # ネットワークをGPUに転送 net = net.to(device) # 学習の再開時はargs['resume']のパラメータをロード if args['resume']: print('Resuming training, loading {}...'.format(args['resume'])) net.load_weights(args['save_folder'] + args['resume']) sources = list() loc = list() conf = list() if args['cuda']: net = torch.nn.DataParallel(net)
## We compute the needed value we need to load in ram only parts of the dataset at the same time with open(TAGS_FULL_PATH, 'r') as f: tags = f.readline().strip().split(',') CLASSES_COUNT = len(tags) with h5py.File(HDF5_FULL_PATH, 'r') as f: DATA_COUNT = len(f['my_labels'][()]) if MAX_DATA_LOAD >= DATA_COUNT: DATA_SPLIT = 0 else: DATA_SPLIT = math.ceil(DATA_COUNT / MAX_DATA_LOAD) COUNT_SPLIT = DATA_COUNT // DATA_SPLIT #1# Original InceptionV3 model loading model = net.build_model(CLASSES_COUNT) model.compile( optimizer='rmsprop', loss='categorical_crossentropy', metrics=[metrics.categorical_accuracy, metrics.top_k_categorical_accuracy]) logger.log("The Original InceptionV3 model has been loaded", 0) #2# Train the model on the new data for a few epochs and save logger.log("Model first train, evaluation and save", 0) first_train_start = time.time() filepath = MODEL_FILE_FULL_PATH + "_0.h5" train_model(model, filepath) net.save(model, tags, MODEL_FILE_FULL_PATH + "_0") logger.execution_time(first_train_start,
def main(_): model = build_model() model.load_weights(FLAGS.input_ckpt) model.save_weights(FLAGS.output_ckpt, save_format='tf')
image = image.transpose((1, 2, 0)) image = scipy.misc.imresize(image, (image_size, image_size)).astype(np.uint8) tilepos_x = bucket_size * predicted_tag tilepos_y = bucket_size * actual_tag tilepos_x += example_count % bucket_size tilepos_y += example_count // bucket_size pos_x, pos_y = tilepos_x * image_size, tilepos_y * image_size vis_image[pos_y:pos_y+image_size, pos_x:pos_x+image_size, :] = image example_counts[(predicted_tag, actual_tag)] += 1 vis_image[::image_size * bucket_size, :] = 0 vis_image[:, ::image_size * bucket_size] = 0 scipy.misc.imsave(vis_filename, vis_image) print "loading original inception model" model = net.build_model(nb_classes) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=["accuracy"]) # train the model on the new data for a few epochs print "training the newly added dense layers, train=%d test=%d batch/%d" % (X_train.shape[0], X_test.shape[0], batch_size) model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size, shuffle=True), steps_per_epoch=X_train.shape[0]/batch_size, epochs=epoch, validation_data=datagen.flow(X_test, Y_test, batch_size=batch_size), validation_steps=X_test.shape[0]/batch_size, callbacks=[TensorBoard(log_dir='./tb_logs')], ) evaluate(model, "000.png")
def init_model(self): model = net.build_model(self.net, self.dataset.nb_classes) model.compile(optimizer=self.optimizer, loss='categorical_crossentropy', metrics=["accuracy"]) self.model = model
# 計算グラフを作成しない with torch.no_grad(): # 順伝播を実行し、推論結果を出力 y = net(x) y = sigm(y) misstooth = [dlabels[i] for i, x in enumerate((y < 0.3).tolist()[0]) if x] ##################### # FRCNN shit ##################### torch.set_default_tensor_type('torch.cuda.FloatTensor') # ネットワークの定義 # 引数が'test'だと、推論結果に対してクラスDetectで後処理を実行 net = build_model('FasterRCNN', 'test', 33) # initialize SSD # GPUへの転送 net = net.to(device) net = torch.nn.DataParallel(net) cudnn.benchmark = True # 学習済みモデルのロード net.load_state_dict(torch.load('./weights/DentalPano_v0.1f.pth')) args = FRCNN_train model = args['model'] dataset = VOCDetection(root=VOC_ROOT, target_transform=args['target_trans'], transform=FRCNNAugmentation())
os.environ["CUDA_VISIBLE_DEVICES"] = '0' #指定第一块GPU可用 config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.6 # 程序最多只能占用指定gpu50%的显存 config.gpu_options.allow_growth = True #程序按需申请内存 sess = tf.Session(config = config) # 设置session KTF.set_session(sess) model1 = model_from_yaml(open('model_architecture_1.yaml').read()) model1.load_weights('model_weights_1.h5') model2 = model_from_yaml(open('model_architecture_2.yaml').read()) model2.load_weights('model_weights_2.h5') input_layer = Input((1280, 1)) output_layer = build_model(input_layer=input_layer,start_neurons=16) model3 = Model(input_layer, output_layer) model3.load_weights('model_weights_3.h5') model4 = model_from_yaml(open('model_architecture_4.yaml').read()) model4.load_weights('model_weights_4.h5') model = [] model.append(model1) model.append(model2) model.append(model3) model.append(model4) def load_ans(data_path_, rpos_path_, fs_): '''