def build(settings={}):
gpu_num = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
settings['dev'] = '/gpu:0'
net_dict_lst = []
net_dict_lst.append(net.build_model(settings['net'], settings))
train_op = net_dict_lst[-1]['opt' + settings['loss']]
if gpu_num > 1:
tower_grads = []
with tf.device('/cpu:0'):
opt = tf.train.AdamOptimizer(net_dict_lst[0]['lr'])
with tf.device(settings['dev']):
grads = opt.compute_gradients(net_dict_lst[0][settings['loss']])
tower_grads.append(grads)
for i in range(1, gpu_num):
settings['dev'] = '/gpu:%d' % i
settings['reuse'] = True
net_dict_lst.append(net.build_model(settings['net'], settings))
with tf.device(settings['dev']):
grads = opt.compute_gradients(
net_dict_lst[0][settings['loss']])
tower_grads.append(grads)
with tf.device('/gpu:0'):
avg_grads = average_gradients(tower_grads)
with tf.device('/cpu:0'):
train_op = opt.apply_gradients(
avg_grads, global_step=net_dict_lst[0][settings['step']])
print(settings['net'] + ' built on %d gpus' % gpu_num)
return net_dict_lst, train_op
def main(args):
config.merge_from_list(args.opts)
cfg, logger = default_setup(config, args)
if args.debug:
batches = int(cfg.SOLVER.IMS_PER_DEVICE * args.num_gpus)
if cfg.SOLVER.IMS_PER_BATCH != batches:
cfg.SOLVER.IMS_PER_BATCH = batches
logger.warning(
"SOLVER.IMS_PER_BATCH is changed to {}".format(batches))
valid_files = get_valid_files(args, cfg, logger)
# * means all if need specific format then *.csv
for current_file in valid_files:
cfg.MODEL.WEIGHTS = current_file
model = build_model(cfg)
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
if cfg.TEST.AUG.ENABLED:
res = Trainer.test_with_TTA(cfg, model)
else:
res = Trainer.test(cfg, model)
if comm.is_main_process():
verify_results(cfg, res)
def main(args):
config.merge_from_list(args.opts)
cfg, logger = default_setup(config, args)
model = build_model(cfg)
logger.info(f"Model structure: {model}")
file_sys = os.statvfs(cfg.OUTPUT_DIR)
free_space_Gb = (file_sys.f_bfree * file_sys.f_frsize) / 2**30
# We assume that a single dumped model is 700Mb
eval_space_Gb = (cfg.SOLVER.LR_SCHEDULER.MAX_ITER //
cfg.SOLVER.CHECKPOINT_PERIOD) * 700 / 2**10
if eval_space_Gb > free_space_Gb:
logger.warning(f"{Fore.RED}Remaining space({free_space_Gb}GB) "
f"is less than ({eval_space_Gb}GB){Style.RESET_ALL}")
if args.eval_only:
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
res = Trainer.test(cfg, model)
if comm.is_main_process():
verify_results(cfg, res)
if cfg.TEST.AUG.ENABLED:
res.update(Trainer.test_with_TTA(cfg, model))
return res
"""
If you'd like to do anything fancier than the standard training logic,
consider writing your own training loop or subclassing the trainer.
"""
trainer = Trainer(cfg, model)
trainer.resume_or_load(resume=args.resume)
if cfg.TEST.AUG.ENABLED:
trainer.register_hooks([
hooks.EvalHook(0,
lambda: trainer.test_with_TTA(cfg, trainer.model))
])
return trainer.train()
def debug_train():
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
traindir = "/data4T1/samhu/shapenet_split_complete/train"
settings = {}
settings['batch_size'] = 32
settings['height'] = 192
settings['width'] = 256
net_name = 'KPARAM_33_DUAL'
net_dict = net.build_model(net_name, settings)
train_fetcher = DataFetcher()
train_fetcher.BATCH_SIZE = settings['batch_size']
train_fetcher.PTS_DIM = 3
train_fetcher.HEIGHT = settings['height']
train_fetcher.WIDTH = settings['width']
train_fetcher.Dir = util.listdir(traindir, ".h5")
train_fetcher.shuffleDir()
if 'rand' in net_dict.keys():
train_fetcher.randfunc = net_dict['rand']
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
lrate = 3e-5
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
try:
train_fetcher.start()
lastEpoch = 0
data_dict = train_fetcher.fetch()
x2D = data_dict['x2D']
x3D = data_dict['x3D']
yGT = 0.8 * data_dict['x3D_final']
GT_PTS_NUM = int(yGT.shape[1])
print x3D.shape
print yGT.shape
yGT = yGT.reshape((-1, 3))
x3D = x3D.reshape((-1, 3))
feed = {
net_dict['yGT']: yGT,
net_dict['ix3D']: x3D,
net_dict['ix2D']: x2D,
net_dict['lr']: lrate
}
if 'eidx' in data_dict.keys():
i = 0
while 'eidx_%d' % i in net_dict.keys():
feed[net_dict['eidx_%d' % i]] = data_dict['eidx'][i]
i += 1
if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()):
feed[net_dict['fidx']] = data_dict['fidx'][-1]
yout = sess.run(net_dict['ox3D'], feed_dict=feed)
print yout.shape
train_fetcher.shutdown()
finally:
train_fetcher.shutdown()
return
def main(args):
config.merge_from_list(args.opts)
cfg, logger = default_setup(config, args)
if args.debug:
batches = int(cfg.SOLVER.IMS_PER_BATCH / 8 * args.num_gpus)
if cfg.SOLVER.IMS_PER_BATCH != batches:
cfg.SOLVER.IMS_PER_BATCH = batches
logger.warning(
"SOLVER.IMS_PER_BATCH is changed to {}".format(batches))
if "MODEL.WEIGHTS" in args.opts:
if cfg.MODEL.WEIGHTS.endswith(".pth") and not PathManager.exists(
cfg.MODEL.WEIGHTS):
ckpt_name = cfg.MODEL.WEIGHTS.split("/")[-1]
model_prefix = cfg.OUTPUT_DIR.split("cvpods_playground")[1][1:]
remote_file_path = os.path.join(cfg.OSS.DUMP_PREFIX, model_prefix,
ckpt_name)
logger.warning(
f"The specified ckpt file ({cfg.MODEL.WEIGHTS}) was not found locally,"
f" try to load the corresponding dump file on OSS ({remote_file_path})."
)
cfg.MODEL.WEIGHTS = remote_file_path
valid_files = [cfg.MODEL.WEIGHTS]
else:
list_of_files = glob.glob(os.path.join(cfg.OUTPUT_DIR, '*.pth'))
assert list_of_files, "No checkpoint file found in {}.".format(
cfg.OUTPUT_DIR)
list_of_files.sort(key=os.path.getctime)
latest_file = list_of_files[-1]
if not args.end_iter:
valid_files = [latest_file]
else:
files = [f for f in list_of_files if str(f) <= str(latest_file)]
valid_files = []
for f in files:
try:
model_iter = int(re.split(r'(model_|\.pth)', f)[-3])
except Exception:
logger.warning("remove {}".format(f))
continue
if args.start_iter <= model_iter <= args.end_iter:
valid_files.append(f)
assert valid_files, "No .pth files satisfy your requirement"
# * means all if need specific format then *.csv
for current_file in valid_files:
cfg.MODEL.WEIGHTS = current_file
model = build_model(cfg)
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
res = Trainer.test(cfg, model)
if comm.is_main_process():
verify_results(cfg, res)
if cfg.TEST.AUG.ENABLED:
res.update(Trainer.test_with_TTA(cfg, model))
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
tf.keras.backend.set_learning_phase(0)
model = build_model()
model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy())
graph_def = K.get_session().graph.as_graph_def()
graph_def = graph_util.extract_sub_graph(graph_def, [FLAGS.output_nodes])
tf.train.write_graph(graph_def,
FLAGS.save_dir,
FLAGS.graph_filename,
as_text=True)
print("Finish export inference graph: {}".format(FLAGS.save_dir))
def main():
parser = argparse.ArgumentParser(
description='Example of Keras Data Generator')
parser.add_argument('--train_dir', default='data/cifar/train/')
parser.add_argument('--test_dir', default='data/cifar/train/')
parser.add_argument('--label_file', default='data/cifar/labels.txt')
parser.add_argument('--batchsize',
'-b',
type=int,
default=32,
help='Number of images in each mini-batch')
parser.add_argument('--epoch',
'-e',
type=int,
default=100,
help='Number of sweeps over the dataset to train')
parser.add_argument('--out',
'-o',
default='result',
help='Directory to output the result')
args = parser.parse_args()
print('batchsize: {}'.format(args.batchsize))
print('epoch: {}'.format(args.epoch))
# Datasets
train_dir = pathlib.Path(args.train_dir)
train_datagen = ImageDataGenerator()
test_dir = pathlib.Path(args.test_dir)
test_datagen = ImageDataGenerator()
classes = list(pd.read_csv(args.label_file, header=-1)[0])
input_shape = (32, 32, 3)
# Model
model = build_model(input_shape, len(classes))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.9),
metrics=['accuracy'])
model.summary()
# Train
model.fit_generator(
generator=train_datagen.flow_from_directory(train_dir, classes),
steps_per_epoch=int(
np.ceil(len(list(train_dir.iterdir())) / args.batchsize)),
epochs=args.epoch,
verbose=1,
validation_data=test_datagen.flow_from_directory(test_dir, classes),
validation_steps=int(
np.ceil(len(list(test_dir.iterdir())) / args.batchsize)))
def main(args):
config.merge_from_list(args.opts)
cfg, logger = default_setup(config, args)
if args.debug:
batches = int(cfg.SOLVER.IMS_PER_BATCH / 8 * args.num_gpus)
if cfg.SOLVER.IMS_PER_BATCH != batches:
cfg.SOLVER.IMS_PER_BATCH = batches
logger.warning(
"SOLVER.IMS_PER_BATCH is changed to {}".format(batches))
if "MODEL.WEIGHTS" in args.opts:
valid_files = [cfg.MODEL.WEIGHTS]
else:
list_of_files = glob.glob(os.path.join(cfg.OUTPUT_DIR, '*.pth'))
assert list_of_files, "no pth file found in {}".format(cfg.OUTPUT_DIR)
list_of_files.sort(key=os.path.getctime)
latest_file = list_of_files[-1]
if not args.end_iter:
valid_files = [latest_file]
else:
files = [f for f in list_of_files if str(f) <= str(latest_file)]
valid_files = []
for f in files:
try:
model_iter = int(re.split(r'(model_|\.pth)', f)[-3])
except Exception:
logger.warning("remove {}".format(f))
continue
if args.start_iter <= model_iter <= args.end_iter:
valid_files.append(f)
assert valid_files, "No .pth files satisfy your requirement"
# * means all if need specific format then *.csv
for current_file in valid_files:
cfg.MODEL.WEIGHTS = current_file
model = build_model(cfg)
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
res = Trainer.test(cfg, model)
if comm.is_main_process():
verify_results(cfg, res)
if cfg.TEST.AUG.ENABLED:
res.update(Trainer.test_with_TTA(cfg, model))
def main(args):
config.merge_from_list(args.opts)
cfg = setup(args)
model = build_model(cfg)
logger.info("Model:\n{}".format(model))
if args.eval_only:
DefaultCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
return do_test(cfg, model)
distributed = comm.get_world_size() > 1
if distributed:
model = DistributedDataParallel(model,
device_ids=[comm.get_local_rank()],
broadcast_buffers=False)
do_train(cfg, model)
return do_test(cfg, model)
def main(_):
batch_size = 120
epochs = FLAGS.epochs
model = build_model()
model.compile(optimizer=tf.train.AdamOptimizer(0.001),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
if FLAGS.pretrained:
model.load_weights(FLAGS.pretrained)
# Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset.
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1)
score = model.evaluate(x_test, y_test, verbose=1)
model.save_weights(FLAGS.ckpt_path, save_format='tf')
def train_net():
os.environ["CUDA_VISIBLE_DEVICES"] = '0' #指定第一块GPU可用
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.6 # 程序最多只能占用指定gpu50%的显存
config.gpu_options.allow_growth = True #程序按需申请内存
sess = tf.Session(config=config)
# 设置session
KTF.set_session(sess)
output_length = fs * 10
input_length = fs * 10
n_classes = 2
input_length = 1280
for fold in range(1, 6, 1):
print("*******************fold {}*****************".format(fold))
tr_data_path = "./validation/train/validation_{}_data.npy".format(
fold) #validation
tr_label_path = "./validation/train/validation_{}_target.npy".format(
fold)
val_data_path = "./validation/val/validation_{}_data.npy".format(fold)
val_label_path = "./validation/val/validation_{}_target.npy".format(
fold) #cv
X_tr = np.load(tr_data_path)
y_tr = np.load(tr_label_path).reshape(-1, 1280, 1)
X_val = np.load(val_data_path)
y_val = np.load(val_label_path).reshape(-1, 1280, 1)
# callback_lists
checkpoint = ModelCheckpoint(filepath="./model/"+ \
'model_weights_mse_0.08_1280_{}_fold_{}_ys.h5' \
.format(time.strftime('%Y-%m-%d %X').split(" ")[0],fold),
monitor= 'val_loss',
mode='min',
verbose=1,
save_best_only='True',
save_weights_only='True')
csv_logger = CSVLogger(
filename='./log/log{}_fold{}.csv'.format( \
time.strftime('%Y-%m-%d %X').split(" ")[0],fold),
separator=',',
append=True)
earlystop = EarlyStopping(
monitor='val_loss',
min_delta=0,
patience=5,
verbose=1,
mode="min",
#baseline=None,
#restore_best_weights=True,
)
reducelr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=3,
verbose=1,
min_lr=1e-5)
callback_lists = [earlystop, checkpoint, reducelr]
input_layer = Input((input_length, 1))
output_layer = build_model(input_layer=input_layer,
block="resnext",
start_neurons=16,
DropoutRatio=0.5,
filter_size=32,
nClasses=2)
model = Model(input_layer, output_layer)
#print(model.summary())
model.compile(
loss='mse', #'categorical_crossentropy',
optimizer=RAdam(1e-4), #RAdam(lr_schedule(0)),#
metrics=['accuracy', 'mse', 'mae']) #focal_loss #mape
history = model.fit(
X_tr,
y_tr,
epochs=400, #350 train
batch_size=32,
verbose=2,
validation_data=(X_val, y_val),
callbacks=callback_lists) #class_weight=class_weight
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=0,
width_shift_range=0.125,
height_shift_range=0.125,
horizontal_flip=True,
vertical_flip=False,
fill_mode='nearest')
test_datagen = ImageDataGenerator(rescale=1. / 255)
print "loading original inception model"
model = net.build_model(nb_classes)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=["accuracy"])
# train the model on the new data for a few epochs
print "training the newly added dense layers"
net.save(model, model_file_prefix)
# we chose to train the top 2 inception blocks, i.e. we will freeze
# the first 172 layers and unfreeze the rest:
for layer in model.layers[:172]:
layer.trainable = False
for layer in model.layers[172:]:
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=0,
width_shift_range=0.125,
height_shift_range=0.125,
horizontal_flip=True,
vertical_flip=False,
fill_mode='nearest',
)
logger.execution_time(dataset_start, "Dataset gathering and formating", 0)
#2# Original InceptionV3 model loading
model = net.build_model(classes_count)
model.compile(
optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=[metrics.categorical_accuracy, metrics.top_k_categorical_accuracy])
logger.log("The Original InceptionV3 model has been loaded", 0)
#3# Train the model on the new data for a few epochs and save
logger.log("Model first train, evaluation and save", 0)
first_train_start = time.time()
filepath = MODEL_FILE_FULL_PATH + "_0.h5"
# Cross validation loop
for i, (train_index, test_index) in enumerate(skf.split(data, y)):
logger.log("Folds " + str(i), 2)
def train():
INPUT_DATA = 'D:/Project/classify/demo/dataset/age/mtcnn_hunhe_train/'
test_DATA = 'D:/Project/classify/demo/dataset/age/mtcnn_hunhe_train/'
batch_size = 32
# initial_learning_rate = 1e-4
train_datagen = ImageDataGenerator(
# preprocessing_function=preprocess_input_new, # ((x/255)-0.5)*2 归一化到±1之间
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.5,
horizontal_flip=True,
)
train_generator = train_datagen.flow_from_directory(directory=INPUT_DATA,
target_size=(image_size, image_size), # Inception V3规定大小
batch_size=batch_size)
print(train_generator.class_indices)
val_generator = train_datagen.flow_from_directory(directory=test_DATA,
target_size=(image_size, image_size), # Inception V3规定大小
batch_size=batch_size)
model = net.build_model(8, (image_size, image_size, 3))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[myacc])
# model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
# metrics: 评价函数, 与损失函数类似, 只不过评价函数的结果不会用于训练过程中, 可以传递已有的评价函数名称, 或者传递一个自定义的theano / tensorflow函数来使用, 自带的评价函数有: binary_accuracy(
# y_true, y_pred), categorical_accuracy(y_true, y_pred), sparse_categorical_accuracy(y_true,
# y_pred), top_k_categorical_accuracy(y_true, y_pred, k=5).自定义评价函数应该在编译的时候compile传递进去,
# 该函数需要以(y_true, y_pred) 作为输入参数, 并返回一个张量作为输出结果.
try:
os.mkdir("./model/")
except OSError:
print(OSError)
pass
checkpoint = ModelCheckpoint(filepath='./model/weights.{epoch:02d}.hdf5',
monitor='val_loss',
verbose=1,
save_weights_only=False,
period=1)
path = 'D:/Project/classify/demo/dataset/age/mtcnn_hunhe_test/'
testmodel = TestModel(model, path, train_generator)
# lr_scheduler = LearningRateScheduler(ir_Schedule)
# optimizer = Adam(lr=initial_learning_rate, decay=1e-6)
# model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=[acc])
patience = 50
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
patience=int(patience / 4), verbose=1)
# 可以自己定义compile函数中传入的myacc，也可以用系统定义的 acc,lr,loss,val_loss,val_acc，来作为检测项目。
callbacks = [checkpoint, reduce_lr, testmodel]
model.fit_generator(generator=train_generator,
steps_per_epoch=int(train_generator.samples / batch_size), #参数steps_per_epoch是通过把训练图像的数量除以批次大小得出的。
epochs=50, # 例如，有100张图像且批次大小为50，则steps_per_epoch值为2。参数epoch决定网络中所有图像的训练次数。
validation_data=val_generator,
validation_steps=1,
class_weight='auto', callbacks=callbacks)
model.save('./models/%s.h5' % model_name)
del train_generator
del val_generator
del model
gc.collect()
def testlayers(settings={}):
if not os.path.exists(preddir):
os.mkdir(preddir);
net_model = None;
config = None;
if not os.environ["CUDA_VISIBLE_DEVICES"]:
net_dict = net.build_model(net_name,settings);
config = tf.ConfigProto(intra_op_parallelism_threads=4,device_count={'gpu':0});
else:
net_dict = net.build_model(net_name,settings);
config = tf.ConfigProto();
config.gpu_options.allow_growth = True;
config.allow_soft_placement = True;
test_fetcher = DataFetcher();
test_fetcher.BATCH_SIZE = settings['batch_size'];
test_fetcher.PTS_DIM = 3;
test_fetcher.HEIGHT = settings['height'];
test_fetcher.WIDTH = settings['width'];
test_fetcher.Dir = util.listdir(traindir,".h5");
test_fetcher.useMix = False;
FetcherLst.append(test_fetcher);
if 'rand' in net_dict.keys():
test_fetcher.randfunc=net_dict['rand'];
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer());
saver = tf.train.Saver();
ckpt = tf.train.get_checkpoint_state('%s/'%dumpdir);
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path);
else:
print "failed to restore model";
return;
layers = [];
li = 0;
while 'ox3D%02d'%li in net_dict.keys():
layers.append(net_dict['ox3D%02d'%li]);
li += 1;
cnt = 30;
test_fetcher.Cnt = cnt;
try:
test_fetcher.start();
data_dict = test_fetcher.fetch();
x2D = data_dict['x2D'];
x3D = data_dict['x3D'];
yGT = data_dict['yGT'];
tag = test_fetcher.fetchTag();
#
yGTout = yGT.copy();
yGT = yGT.reshape((-1,3));
x3D = x3D.reshape((-1,3));
feed={
net_dict['yGT']:yGT,
net_dict['ix3D']:x3D,
net_dict['ix2D']:x2D
};
if 'eidx' in data_dict.keys():
i = 0;
while 'eidx_%d'%i in net_dict.keys():
feed[net_dict['eidx_%d'%i]] = data_dict['eidx'][i];
i += 1;
if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()):
feed[net_dict['fidx']] = data_dict['fidx'][-1];
ylayers = sess.run(layers,feed_dict=feed);
fdir = preddir+os.sep+"pred_%s_%03d"%(tag,cnt);
if not os.path.exists(fdir):
os.mkdir(fdir);
i = 0;
for layer in layers:
lname = layer.name.replace(':','_');
if i < len(data_dict['fidx']):
fidx = data_dict['fidx'][i];
else:
fidx = data_dict['fidx'][-1];
f_lst = [];
for j in range(test_fetcher.BATCH_SIZE):
f_lst.append(fidx);
util.write_to_obj(fdir+os.sep+"obj%02d"%i,ylayers[i],faces=f_lst);
i += 1;
finally:
test_fetcher.shutdown();
return;
def test(settings={}):
if not os.path.exists(preddir):
os.mkdir(preddir);
net_model = None;
config = None;
if not os.environ["CUDA_VISIBLE_DEVICES"]:
net_dict = net.build_model(net_name,settings);
config = tf.ConfigProto(intra_op_parallelism_threads=4,device_count={'gpu':0});
else:
net_dict = net.build_model(net_name,settings);
config = tf.ConfigProto();
config.gpu_options.allow_growth = True;
config.allow_soft_placement = True;
test_fetcher = DataFetcher();
test_fetcher.BATCH_SIZE = settings['batch_size'];
test_fetcher.PTS_DIM = 3;
test_fetcher.HEIGHT = settings['height'];
test_fetcher.WIDTH = settings['width'];
test_fetcher.Dir = util.listdir(traindir,".h5");
test_fetcher.useMix = False;
FetcherLst.append(test_fetcher);
if 'rand' in net_dict.keys():
test_fetcher.randfunc=net_dict['rand'];
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer());
saver = tf.train.Saver();
ckpt = tf.train.get_checkpoint_state('%s/'%dumpdir);
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path);
else:
print "failed to restore model";
return;
stat = {};
try:
test_fetcher.start();
for cnt in range(len(test_fetcher.Dir)):
data_dict = test_fetcher.fetch();
x2D = data_dict['x2D'];
x3D = data_dict['x3D'];
yGT = data_dict['yGT'];
tag = test_fetcher.fetchTag();
r2D = None;
if 'r2D' in net_dict.keys():
r2D_dim = int(net_dict['r2D'].shape[1]);
r2D = np.random.normal(loc=0.0,scale=1.0,size=[BATCH_SIZE,r2D_dim]);
#
rgb = None;
f_lst = [];
if 'x3D_final' in data_dict.keys():
rgb = util.sphere_to_YIQ( data_dict['x3D_final'] );
#
yGTout = yGT.copy();
yGT = yGT.reshape((-1,3));
x3D = x3D.reshape((-1,3));
feed={
net_dict['yGT']:yGT,
net_dict['ix3D']:x3D,
net_dict['ix2D']:x2D
};
if 'eidx' in data_dict.keys():
i = 0;
while 'eidx_%d'%i in net_dict.keys():
feed[net_dict['eidx_%d'%i]] = data_dict['eidx'][i];
i += 1;
if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()):
feed[net_dict['fidx']] = data_dict['fidx'][-1];
yout=None;
loss=None;
yout,loss = sess.run([net_dict['ox3D'],net_dict['chmf']],feed_dict=feed);
fdir = preddir+os.sep+"pred_%s_%03d_%f"%(tag,cnt,loss);
#
if not os.path.exists(fdir):
os.mkdir(fdir);
if 'fidx' in data_dict.keys():
for j in range(yout.shape[0]):
f_lst.append(data_dict['fidx'][-1]);
util.write_to_obj(fdir+os.sep+"obj",yout,rgb,f_lst);
else:
util.write_to_obj(fdir+os.sep+"obj",yout);
util.write_to_obj(fdir+os.sep+"GTobj",yGTout);
#generating dense result
util.write_to_img(fdir,x2D);
finally:
test_fetcher.shutdown();
return;
# 計算グラフを作成しない
with torch.no_grad():
# 順伝播を実行し、推論結果を出力
y = net(x)
y = sigm(y)
misstooth = [dlabels[i] for i, x in enumerate((y < 0.3).tolist()[0]) if x]
#####################
# SSD shit
#####################
torch.set_default_tensor_type('torch.cuda.FloatTensor')
# ネットワークの定義
# 引数が'test'だと、推論結果に対してクラスDetectで後処理を実行
ssd_net = build_model('SSD', 33) # initialize SSD
# GPUへの転送
net = ssd_net.to(device)
# 学習済みモデルのロード
net.load_weights('./weights/DentalPano_v0.6(CLAHE, Adam).pth')
dataset = VOCDetection(root=VOC_ROOT,
target_transform=args['target_trans'],
transform=SSDAugmentation())
data_loader = data.DataLoader(dataset, args['batch_size'],
num_workers=args['num_workers'],
shuffle=True,
import torch.utils.data as data
import matplotlib.patches as patches
import matplotlib.pyplot as plt
# TODO: YO HERE!
args = FRCNN_train
model = args['model']
dataset = VOCDetection(root=VOC_ROOT,
target_transform=args['target_trans'],
transform=FRCNNAugmentation())
# ネットワークの定義
net = build_model(model, voc_config['num_classes'])
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# ネットワークをGPUに転送
net = net.to(device)
# 学習の再開時はargs['resume']のパラメータをロード
if args['resume']:
print('Resuming training, loading {}...'.format(args['resume']))
net.load_weights(args['save_folder'] + args['resume'])
sources = list()
loc = list()
conf = list()
if args['cuda']:
net = torch.nn.DataParallel(net)
## We compute the needed value we need to load in ram only parts of the dataset at the same time
with open(TAGS_FULL_PATH, 'r') as f:
tags = f.readline().strip().split(',')
CLASSES_COUNT = len(tags)
with h5py.File(HDF5_FULL_PATH, 'r') as f:
DATA_COUNT = len(f['my_labels'][()])
if MAX_DATA_LOAD >= DATA_COUNT:
DATA_SPLIT = 0
else:
DATA_SPLIT = math.ceil(DATA_COUNT / MAX_DATA_LOAD)
COUNT_SPLIT = DATA_COUNT // DATA_SPLIT
#1# Original InceptionV3 model loading
model = net.build_model(CLASSES_COUNT)
model.compile(
optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=[metrics.categorical_accuracy, metrics.top_k_categorical_accuracy])
logger.log("The Original InceptionV3 model has been loaded", 0)
#2# Train the model on the new data for a few epochs and save
logger.log("Model first train, evaluation and save", 0)
first_train_start = time.time()
filepath = MODEL_FILE_FULL_PATH + "_0.h5"
train_model(model, filepath)
net.save(model, tags, MODEL_FILE_FULL_PATH + "_0")
logger.execution_time(first_train_start,
def main(_):
model = build_model()
model.load_weights(FLAGS.input_ckpt)
model.save_weights(FLAGS.output_ckpt, save_format='tf')
image = image.transpose((1, 2, 0))
image = scipy.misc.imresize(image, (image_size, image_size)).astype(np.uint8)
tilepos_x = bucket_size * predicted_tag
tilepos_y = bucket_size * actual_tag
tilepos_x += example_count % bucket_size
tilepos_y += example_count // bucket_size
pos_x, pos_y = tilepos_x * image_size, tilepos_y * image_size
vis_image[pos_y:pos_y+image_size, pos_x:pos_x+image_size, :] = image
example_counts[(predicted_tag, actual_tag)] += 1
vis_image[::image_size * bucket_size, :] = 0
vis_image[:, ::image_size * bucket_size] = 0
scipy.misc.imsave(vis_filename, vis_image)
print "loading original inception model"
model = net.build_model(nb_classes)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=["accuracy"])
# train the model on the new data for a few epochs
print "training the newly added dense layers, train=%d test=%d batch/%d" % (X_train.shape[0], X_test.shape[0], batch_size)
model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size, shuffle=True),
steps_per_epoch=X_train.shape[0]/batch_size,
epochs=epoch,
validation_data=datagen.flow(X_test, Y_test, batch_size=batch_size),
validation_steps=X_test.shape[0]/batch_size,
callbacks=[TensorBoard(log_dir='./tb_logs')],
)
evaluate(model, "000.png")
def init_model(self):
model = net.build_model(self.net, self.dataset.nb_classes)
model.compile(optimizer=self.optimizer,
loss='categorical_crossentropy',
metrics=["accuracy"])
self.model = model
# 計算グラフを作成しない
with torch.no_grad():
# 順伝播を実行し、推論結果を出力
y = net(x)
y = sigm(y)
misstooth = [dlabels[i] for i, x in enumerate((y < 0.3).tolist()[0]) if x]
#####################
# FRCNN shit
#####################
torch.set_default_tensor_type('torch.cuda.FloatTensor')
# ネットワークの定義
# 引数が'test'だと、推論結果に対してクラスDetectで後処理を実行
net = build_model('FasterRCNN', 'test', 33) # initialize SSD
# GPUへの転送
net = net.to(device)
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
# 学習済みモデルのロード
net.load_state_dict(torch.load('./weights/DentalPano_v0.1f.pth'))
args = FRCNN_train
model = args['model']
dataset = VOCDetection(root=VOC_ROOT,
target_transform=args['target_trans'],
transform=FRCNNAugmentation())
os.environ["CUDA_VISIBLE_DEVICES"] = '0' #指定第一块GPU可用
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.6 # 程序最多只能占用指定gpu50%的显存
config.gpu_options.allow_growth = True #程序按需申请内存
sess = tf.Session(config = config)
# 设置session
KTF.set_session(sess)
model1 = model_from_yaml(open('model_architecture_1.yaml').read())
model1.load_weights('model_weights_1.h5')
model2 = model_from_yaml(open('model_architecture_2.yaml').read())
model2.load_weights('model_weights_2.h5')
input_layer = Input((1280, 1))
output_layer = build_model(input_layer=input_layer,start_neurons=16)
model3 = Model(input_layer, output_layer)
model3.load_weights('model_weights_3.h5')
model4 = model_from_yaml(open('model_architecture_4.yaml').read())
model4.load_weights('model_weights_4.h5')
model = []
model.append(model1)
model.append(model2)
model.append(model3)
model.append(model4)
def load_ans(data_path_, rpos_path_, fs_):
'''
