def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', '-b', type=int, default=-1,
help='each batch size will be a new file')
parser.add_argument('--gpu', '-g', type=int, default=0,
help='gpu that used to extract feature')
parser.add_argument("--out_dir", '-o', default="/home/machen/dataset/new_graph/")
parser.add_argument("--model",'-m', help="the AU R-CNN pretrained model file to load to extract feature")
parser.add_argument("--trainval_test", '-tt', help="train or test")
parser.add_argument("--database", default="BP4D")
parser.add_argument('--use_memcached', action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument('--force_write', action='store_true')
parser.add_argument('--mean', default=config.ROOT_PATH + "BP4D/idx/mean_no_enhance.npy",
help='image mean .npy file')
parser.add_argument('--jump_exist_file', action='store_true',
help='image mean .npy file')
args = parser.parse_args()
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(args.memcached_host))
result_dict = extract_mode(args.model)
fold = result_dict["fold"]
backbone = result_dict["backbone"]
split_idx = result_dict["split_idx"]
if backbone == 'vgg':
faster_rcnn = FasterRCNNVGG16(n_fg_class=len(config.AU_SQUEEZE),
pretrained_model="imagenet",
mean_file=args.mean,
use_lstm=False,
extract_len=1000,
fix=False) # 可改为/home/nco/face_expr/result/snapshot_model.npz
elif backbone == 'resnet101':
faster_rcnn = FasterRCNNResnet101(n_fg_class=len(config.AU_SQUEEZE),
pretrained_model=backbone,
mean_file=args.mean,
use_lstm=False,
extract_len=1000, fix=False)
assert os.path.exists(args.model)
print("loading model file : {}".format(args.model))
chainer.serializers.load_npz(args.model, faster_rcnn)
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
faster_rcnn.to_gpu(args.gpu)
dataset = AUExtractorDataset(database=args.database,
fold=fold, split_name=args.trainval_test,
split_index=split_idx, mc_manager=mc_manager, use_lstm=False,
train_all_data=False,
prefix="", pretrained_target="", pretrained_model=faster_rcnn, extract_key="avg_pool",
device=-1, batch_size=args.batch_size
)
train_test = "train" if args.trainval_test == "trainval" else "test"
jump_dataset = JumpExistFileDataset(dataset, args.out_dir, fold, args.database, split_idx,
args.batch_size, train_test, args.jump_exist_file)
dataset_iter = BatchKeepOrderIterator(jump_dataset, batch_size=args.batch_size, repeat=False, shuffle=False)
file_key_counter = 0
last_sequence_key = None
for batch in dataset_iter:
features = []
bboxes = []
labels = []
file_key_counter += 1
for idx, (feature, bbox, label, img_path, _file_key_counter) in enumerate(batch):
sequence_key = "_".join((img_path.split("/")[-3], img_path.split("/")[-2]))
if last_sequence_key is None:
last_sequence_key = sequence_key
if sequence_key!=last_sequence_key:
file_key_counter = 1
last_sequence_key = sequence_key
assert file_key_counter == _file_key_counter, (file_key_counter, _file_key_counter, img_path)
if feature is None:
print("jump img_path : {}".format(img_path))
continue
features.extend(feature)
bboxes.extend(bbox)
labels.extend(label)
if features:
if args.trainval_test == "trainval":
file_name = args.out_dir + os.sep + "{0}_{1}_fold_{2}".format(args.database,fold, split_idx) + "/train" +os.sep +sequence_key + "@" + str(file_key_counter) + ".npz"
else:
file_name = args.out_dir + os.sep + "{0}_{1}_fold_{2}".format(args.database,fold, split_idx) + "/test" + os.sep +sequence_key + "@" + str(file_key_counter) + ".npz"
os.makedirs(os.path.dirname(file_name), exist_ok=True)
features = np.stack(features)
bboxes = np.stack(bboxes)
labels = np.stack(labels)
print("write : {}".format(file_name))
assert not os.path.exists(file_name), file_name
np.savez(file_name, feature=features, bbox=bboxes, label=labels)
def main():
parser = argparse.ArgumentParser(description='I3D R-CNN train:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='i3d_result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--mean',
default=config.ROOT_PATH +
"BP4D/idx/mean_no_enhance.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="mobilenet_v1",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='SGD',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_rgb',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument(
'--pretrained_flow',
help=
"path of optical flow pretrained model (may be single stream OF model)"
)
parser.add_argument('--two_stream_mode',
type=TwoStreamMode,
choices=list(TwoStreamMode),
help='spatial/ temporal/ spatial_temporal')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument(
"--roi_align",
action="store_true",
help="whether to use roi align or roi pooling layer in CNN")
parser.add_argument("--T",
'-T',
type=int,
default=10,
help="sequence length of one video clip")
parser.add_argument("--out_channel",
type=int,
default=2048,
help="length of extract ROI feature")
parser.add_argument("--proc_num", "-proc", type=int, default=1)
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
paper_report_label, class_num = squeeze_label_num_report(
args.database, args.use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
au_rcnn_train_chain_list = []
if args.backbone == 'i3d':
if args.two_stream_mode == TwoStreamMode.rgb:
i3d_feature_backbone = I3DFeatureExtractor(modality='rgb')
i3d_roi_head = I3DRoIHead(out_channel=args.out_channel,
roi_size=7,
spatial_scale=1 / 16.,
dropout_prob=0.)
chainer.serializers.load_npz(args.pretrained_flow,
i3d_feature_backbone)
chainer.serializers.load_npz(args.pretrained_flow, i3d_roi_head)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(
i3d_feature_backbone, i3d_roi_head)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
elif args.two_stream_mode == TwoStreamMode.optical_flow:
i3d_feature_backbone_flow = I3DFeatureExtractor(modality='flow')
i3d_roi_head = I3DRoIHead(out_channel=args.out_channel,
roi_size=7,
spatial_scale=1 / 16.,
dropout_prob=0.)
au_rcnn_train_chain_flow = AU_RCNN_ROI_Extractor(
i3d_feature_backbone_flow, i3d_roi_head)
chainer.serializers.load_npz(args.pretrained_flow,
i3d_feature_backbone_flow)
chainer.serializers.load_npz(args.pretrained_flow, i3d_roi_head)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_flow)
elif args.two_stream_mode == TwoStreamMode.rgb_flow:
i3d_feature_backbone = I3DFeatureExtractor(modality='rgb')
i3d_roi_head_rgb = I3DRoIHead(out_channel=args.out_channel,
roi_size=7,
spatial_scale=1 / 16.,
dropout_prob=0.)
chainer.serializers.load_npz(args.pretrained_rgb,
i3d_feature_backbone)
chainer.serializers.load_npz(args.pretrained_rgb, i3d_roi_head_rgb)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(
i3d_feature_backbone, i3d_roi_head_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
i3d_feature_backbone_flow = I3DFeatureExtractor(modality='flow')
i3d_roi_head_flow = I3DRoIHead(out_channel=args.out_channel,
roi_size=7,
spatial_scale=1 / 16.,
dropout_prob=0.)
au_rcnn_train_chain_flow = AU_RCNN_ROI_Extractor(
i3d_feature_backbone_flow, i3d_roi_head_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_flow)
chainer.serializers.load_npz(args.pretrained_flow,
i3d_feature_backbone_flow)
chainer.serializers.load_npz(args.pretrained_flow,
i3d_roi_head_flow)
au_rcnn_train_loss = AU_RCNN_TrainChainLoss()
loss_head_module = au_rcnn_train_loss
model = Wrapper(au_rcnn_train_chain_list, loss_head_module, args.database,
args.T, args.two_stream_mode, args.gpu)
batch_size = args.batch_size
img_dataset = AUDataset(database=args.database,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False)
train_video_data = AU_video_dataset(
au_image_dataset=img_dataset,
sample_frame=args.T,
train_mode=True,
paper_report_label_idx=paper_report_label_idx)
Transform = Transform3D
substract_mean = SubStractMean(args.mean)
train_video_data = TransformDataset(
train_video_data, Transform(substract_mean, mirror=False))
if args.proc_num == 1:
train_iter = SerialIterator(train_video_data,
batch_size * args.sample_frame,
repeat=True,
shuffle=False)
else:
train_iter = MultiprocessIterator(train_video_data,
batch_size=batch_size *
args.sample_frame,
n_processes=args.proc_num,
repeat=True,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
for gpu in args.gpu:
chainer.cuda.get_device_from_id(gpu).use()
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
key_str = "{0}_fold_{1}".format(args.fold, args.split_idx)
file_list = []
file_list.extend(os.listdir(args.out))
# BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz
use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_{}_label".format(
args.database)
roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling"
single_model_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_model.npz'.format(args.database,
args.fold, args.split_idx,
args.backbone, args.two_stream_mode,
use_paper_key_str, roi_align_key_str,
args.T)
print(single_model_file_name)
pretrained_optimizer_file_name = args.out + os.sep +\
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_optimizer.npz'.format(args.database,
args.fold, args.split_idx,
args.backbone, args.two_stream_mode,
use_paper_key_str, roi_align_key_str,
args.T)
print(pretrained_optimizer_file_name)
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model)
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=args.gpu[0],
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
@training.make_extension(trigger=(1, "epoch"))
def reset_order(trainer):
print("reset dataset order after one epoch")
trainer.updater._iterators[
"main"].dataset._dataset.reset_for_train_mode()
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(reset_order)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 10, 'iteration'
plot_interval = 10, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.1, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="log_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.log".format(
args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T)))
# trainer.reporter.add_observer("main_par", model.loss_head_module)
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/accuracy',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name='loss_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/accuracy'],
file_name='accuracy_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type),
trigger=plot_interval),
trigger=plot_interval)
trainer.run()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)'
) # open_crf layer only works for CPU mode
parser.add_argument(
"--model", "-m",
help="pretrained model file path") # which contains pretrained target
parser.add_argument("--pretrained_model", "-pre", default="resnet101")
parser.add_argument("--memcached_host", default="127.0.0.1")
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--proc_num',
type=int,
default=10,
help="multiprocess fetch data process number")
parser.add_argument('--two_stream_mode',
type=TwoStreamMode,
choices=list(TwoStreamMode),
help='spatial/ temporal/ spatial_temporal')
parser.add_argument('--batch',
'-b',
type=int,
default=5,
help='mini batch size')
args = parser.parse_args()
if not args.model.endswith("model.npz"):
return
mode_dict = extract_mode(args.model)
database = mode_dict["database"]
fold = mode_dict["fold"]
split_idx = mode_dict["split_idx"]
backbone = mode_dict["backbone"]
spatial_edge_mode = mode_dict["spatial_edge_mode"]
temporal_edge_mode = mode_dict["temporal_edge_mode"]
use_paper_num_label = mode_dict["use_paper_num_label"]
use_roi_align = mode_dict["use_roi_align"]
use_label_dep_rnn_layer = mode_dict["label_dep_rnn_layer"]
sample_frame = mode_dict["sample_frame"]
conv_rnn_type = mode_dict["conv_rnn_type"]
use_feature_map = (conv_rnn_type != ConvRNNType.conv_rcnn) and (
conv_rnn_type != ConvRNNType.fc_lstm)
use_au_rcnn_loss = (conv_rnn_type == ConvRNNType.conv_rcnn)
adaptive_AU_database(database)
paper_report_label, class_num = squeeze_label_num_report(
database, use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
if not paper_report_label_idx:
paper_report_label_idx = None
class_num = len(config.AU_SQUEEZE)
else:
class_num = len(paper_report_label_idx)
model_print_dict = OrderedDict()
for key, value in mode_dict.items():
model_print_dict[key] = str(value)
print("""
{0}
======================================
INFO:
{1}
======================================
""".format(args.model,
json.dumps(model_print_dict, sort_keys=True, indent=8)))
if backbone == 'resnet101':
au_rcnn = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=use_roi_align,
use_feature_map_res45=use_feature_map,
use_feature_map_res5=(conv_rnn_type != ConvRNNType.fc_lstm or
conv_rnn_type == ConvRNNType.sep_conv_lstm),
temporal_length=sample_frame)
elif backbone == 'resnet50':
au_rcnn = AU_RCNN_Resnet50(pretrained_model=args.pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=use_roi_align,
use_feature_map=use_feature_map)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
# if use_label_dep_rnn_layer:
# use_space = (spatial_edge_mode != SpatialEdgeMode.no_edge)
# use_temporal = (temporal_edge_mode != TemporalEdgeMode.no_temporal)
# label_dependency_layer = LabelDependencyLayer(database, out_size=class_num, train_mode=False,
# label_win_size=2, x_win_size=1,
# label_dropout_ratio=0.0, use_space=use_space,
# use_temporal=use_temporal)
if conv_rnn_type == ConvRNNType.conv_lstm:
space_time_conv_lstm = SpaceTimeConv(
None,
use_label_dep_rnn_layer,
class_num,
spatial_edge_mode=spatial_edge_mode,
temporal_edge_mode=temporal_edge_mode,
conv_rnn_type=conv_rnn_type)
loss_head_module = space_time_conv_lstm
elif conv_rnn_type == ConvRNNType.fc_lstm:
space_time_fc_lstm = SpaceTimeSepFcLSTM(
database,
class_num,
spatial_edge_mode=spatial_edge_mode,
temporal_edge_mode=temporal_edge_mode)
loss_head_module = space_time_fc_lstm
elif conv_rnn_type == ConvRNNType.conv_rcnn:
au_rcnn_train_loss = AU_RCNN_TrainChainLoss()
loss_head_module = au_rcnn_train_loss
elif conv_rnn_type == ConvRNNType.sep_conv_lstm:
space_time_sep_conv_lstm = SpaceTimeSepConv(
database,
class_num,
spatial_edge_mode=spatial_edge_mode,
temporal_edge_mode=temporal_edge_mode)
loss_head_module = space_time_sep_conv_lstm
model = Wrapper(au_rcnn_train_chain,
loss_head_module,
database,
sample_frame,
use_feature_map=use_feature_map,
gpu=args.gpu)
chainer.serializers.load_npz(args.model, model)
print("loading {}".format(args.model))
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu(args.gpu)
mc_manager = PyLibmcManager(args.memcached_host)
img_dataset = AUDataset(
database=database,
fold=fold,
split_name='test', # FIXME
split_index=split_idx,
mc_manager=mc_manager,
train_all_data=False)
video_dataset = AU_video_dataset(
au_image_dataset=img_dataset,
sample_frame=sample_frame,
train_mode=False, #FIXME
paper_report_label_idx=paper_report_label_idx,
fetch_use_parrallel_iterator=True)
video_dataset = TransformDataset(video_dataset,
Transform3D(au_rcnn, mirror=False))
# test_iter = SerialIterator(video_dataset, batch_size=sample_frame * args.batch,
# repeat=False, shuffle=False)
test_iter = MultiprocessIterator(video_dataset,
batch_size=sample_frame * args.batch,
n_processes=args.proc_num,
repeat=False,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
with chainer.no_backprop_mode(), chainer.using_config(
'cudnn_deterministic', True), chainer.using_config('train', False):
npz_path = os.path.dirname(
args.model) + os.path.sep + "pred_" + os.path.basename(
args.model)[:os.path.basename(args.model).rindex("_")] + ".npz"
print("npz_path: {}".format(npz_path))
au_evaluator = ActionUnitEvaluator(
test_iter,
model,
args.gpu,
database=database,
paper_report_label=paper_report_label,
converter=lambda batch, device: concat_examples_not_labels(
batch, device, padding=0),
sample_frame=sample_frame,
output_path=npz_path)
observation = au_evaluator.evaluate()
with open(os.path.dirname(args.model) + os.path.sep + "evaluation_result_{0}.json".format(os.path.basename(args.model)\
[:os.path.basename(args.model).rindex("_")]
), "w") as file_obj:
file_obj.write(
json.dumps(observation, indent=4, separators=(',', ': ')))
file_obj.flush()
def main():
print("chainer cudnn enabled: {}".format(chainer.cuda.cudnn_enabled))
parser = argparse.ArgumentParser(
description='Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
default="0",
help='GPU ID, multiple GPU split by comma, \ '
'Note that BPTT updater do not support multi-GPU')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--readtype', default='rgb', help='rgb/flow')
parser.add_argument('--seed', '-s', type=int, default=0)
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=20)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--feature_model',
default="resnet101",
help="vgg or resnet101 for train")
parser.add_argument('--optimizer',
default='RMSprop',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model',
default='resnet101',
help='imagenet/vggface/resnet101/*.npz')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument(
"--is_pretrained",
action="store_true",
help="whether is to pretrain BP4D later will for DISFA dataset or not")
parser.add_argument(
"--pretrained_target",
'-pt',
default="",
help="whether pretrain label set will use DISFA or not")
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
parser.add_argument('--test_model',
default="",
help='test model for evaluation')
parser.add_argument(
'--occlude',
default='',
help=
'whether to use occlude face of upper/left/right/lower/none to test')
parser.add_argument("--img_resolution", type=int, default=512)
args = parser.parse_args()
config.IMG_SIZE = (args.img_resolution, args.img_resolution)
if not os.path.exists(args.pid):
os.makedirs(args.pid)
pid = str(os.getpid())
pid_file_path = args.pid + os.path.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
print('GPU: {}'.format(args.gpu))
if args.is_pretrained:
adaptive_AU_database(args.pretrained_target)
else:
adaptive_AU_database(args.database)
np.random.seed(args.seed)
# 需要先构造一个list的txt文件:id_trainval_0.txt, 每一行是subject + "/" + emotion_seq + "/" frame
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
resnet101 = ResNet(len(config.AU_SQUEEZE),
pretrained_model=args.pretrained_model)
model = TrainChain(resnet101)
if args.eval_mode:
with chainer.no_backprop_mode(), chainer.using_config("train", False):
if args.occlude:
test_data = ImageDataset(
database=args.database,
fold=args.fold,
split_name='test',
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False,
pretrained_target=args.pretrained_target,
img_resolution=args.img_resolution)
test_data = TransformDataset(
test_data, Transform(mean_rgb_path=args.mean,
mirror=False))
assert args.occlude in ["upper", "lower", "left", "right"]
test_data = TransformDataset(test_data,
OccludeTransform(args.occlude))
if args.proc_num == 1:
test_iter = SerialIterator(test_data,
1,
repeat=False,
shuffle=True)
else:
test_iter = MultiprocessIterator(test_data,
batch_size=1,
n_processes=args.proc_num,
repeat=False,
shuffle=True,
n_prefetch=10,
shared_mem=10000000)
single_model_file_name = args.test_model
chainer.serializers.load_npz(single_model_file_name, resnet101)
gpu = int(args.gpu)
chainer.cuda.get_device_from_id(gpu).use()
resnet101.to_gpu(gpu)
evaluator = AUEvaluator(test_iter,
resnet101,
lambda batch, device: concat_examples(
batch, device, padding=0),
args.database,
"/home/machen/face_expr",
device=gpu,
npz_out_path=args.out + os.path.sep +
"npz_occlude_{0}_split_{1}.npz".format(
args.occlude, args.split_idx))
observation = evaluator.evaluate()
with open(
args.out + os.path.sep +
"evaluation_occlude_{0}_fold_{1}_result_test_mode.json"
.format(args.occlude, args.split_idx),
"w") as file_obj:
file_obj.write(
json.dumps(observation,
indent=4,
separators=(',', ': ')))
file_obj.flush()
else:
test_data = ImageDataset(
database=args.database,
fold=args.fold,
split_name='test',
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False,
pretrained_target=args.pretrained_target,
img_resolution=args.img_resolution)
test_data = TransformDataset(
test_data, Transform(mean_rgb_path=args.mean,
mirror=False))
if args.proc_num == 1:
test_iter = SerialIterator(test_data,
1,
repeat=False,
shuffle=False)
else:
test_iter = MultiprocessIterator(test_data,
batch_size=1,
n_processes=args.proc_num,
repeat=False,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
single_model_file_name = args.test_model
chainer.serializers.load_npz(single_model_file_name, resnet101)
gpu = int(args.gpu) if "," not in args.gpu else int(
args.gpu[:args.gpu.index(",")])
chainer.cuda.get_device_from_id(gpu).use()
resnet101.to_gpu(gpu)
evaluator = AUEvaluator(
test_iter,
resnet101,
lambda batch, device: concat_examples(
batch, device, padding=0),
args.database,
"/home/machen/face_expr",
device=gpu,
npz_out_path=args.out + os.path.sep +
"npz_split_{}.npz".format(args.split_idx))
observation = evaluator.evaluate()
with open(
args.out + os.path.sep +
"evaluation_split_{}_result_train_mode.json".format(
args.split_idx), "w") as file_obj:
file_obj.write(
json.dumps(observation,
indent=4,
separators=(',', ': ')))
file_obj.flush()
return
train_data = ImageDataset(database=args.database,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=args.is_pretrained,
read_type=args.readtype,
pretrained_target=args.pretrained_target,
img_resolution=args.img_resolution)
train_data = TransformDataset(train_data, Transform(args.mean,
mirror=True))
if args.proc_num == 1:
train_iter = SerialIterator(train_data, args.batch_size, True, True)
else:
train_iter = MultiprocessIterator(train_data,
batch_size=args.batch_size,
n_processes=args.proc_num,
repeat=True,
shuffle=True,
n_prefetch=10,
shared_mem=31457280)
if "," in args.gpu:
for gpu in args.gpu.split(","):
chainer.cuda.get_device_from_id(int(gpu)).use()
else:
chainer.cuda.get_device_from_id(int(args.gpu)).use()
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
print("using Adam")
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
print("using AdaDelta")
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
if not os.path.exists(args.out):
os.makedirs(args.out)
pretrained_optimizer_file_name = '{0}_{1}_fold_{2}_{3}_{4}_optimizer.npz'.format(
args.database, args.fold, args.split_idx, args.feature_model,
optimizer_name)
pretrained_optimizer_file_name = args.out + os.path.sep + pretrained_optimizer_file_name
single_model_file_name = args.out + os.path.sep + '{0}_{1}_fold_{2}_{3}_model.npz'.format(
args.database, args.fold, args.split_idx, args.feature_model)
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model.backbone)
print(" GPU({0}) updater".format(args.gpu))
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=int(args.gpu),
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
trainer.extend(chainer.training.extensions.snapshot_object(
model.backbone, filename=os.path.basename(single_model_file_name)),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 100, 'iteration'
plot_interval = 100, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
# use Adam
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.5, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="{0}_fold_{1}.log".format(args.fold, args.split_idx)))
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/accuracy',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss', "validation/main/loss"],
file_name='loss_{0}_fold_{1}.png'.format(args.fold,
args.split_idx),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/accuracy'],
file_name='accuracy_{0}_fold_{1}.png'.format(
args.fold, args.split_idx),
trigger=plot_interval),
trigger=plot_interval)
trainer.run()
def main():
parser = argparse.ArgumentParser(
description='Space Time Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='end_to_end_result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--need_validate',
action='store_true',
help='do or not validate during training')
parser.add_argument('--mean',
default=config.ROOT_PATH +
"BP4D/idx/mean_no_enhance.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="mobilenet_v1",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='SGD',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model_rgb',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument(
'--pretrained_model_of',
help=
"path of optical flow pretrained model (may be single stream OF model)"
)
parser.add_argument('--pretrained_model_args',
nargs='+',
type=float,
help='you can pass in "1.0 224" or "0.75 224"')
parser.add_argument('--spatial_edge_mode',
type=SpatialEdgeMode,
choices=list(SpatialEdgeMode),
help='1:all_edge, 2:configure_edge, 3:no_edge')
parser.add_argument('--spatial_sequence_type',
type=SpatialSequenceType,
choices=list(SpatialSequenceType),
help='1:all_edge, 2:configure_edge, 3:no_edge')
parser.add_argument(
'--temporal_edge_mode',
type=TemporalEdgeMode,
choices=list(TemporalEdgeMode),
help='1:rnn, 2:attention_block, 3.point-wise feed forward(no temporal)'
)
parser.add_argument('--two_stream_mode',
type=TwoStreamMode,
choices=list(TwoStreamMode),
help='spatial/ temporal/ spatial_temporal')
parser.add_argument('--conv_rnn_type',
type=ConvRNNType,
choices=list(ConvRNNType),
help='conv_lstm or conv_sru')
parser.add_argument("--bi_lstm",
action="store_true",
help="whether to use bi-lstm as Edge/Node RNN")
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--layers", type=int, default=1)
parser.add_argument("--label_win_size", type=int, default=3)
parser.add_argument("--fix",
action="store_true",
help="fix parameter of conv2 update when finetune")
parser.add_argument("--x_win_size", type=int, default=1)
parser.add_argument("--use_label_dependency",
action="store_true",
help="use label dependency layer after conv_lstm")
parser.add_argument("--dynamic_backbone",
action="store_true",
help="use dynamic backbone: conv lstm as backbone")
parser.add_argument("--ld_rnn_dropout", type=float, default=0.4)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument(
"--roi_align",
action="store_true",
help="whether to use roi align or roi pooling layer in CNN")
parser.add_argument("--debug",
action="store_true",
help="debug mode for 1/50 dataset")
parser.add_argument("--sample_frame", '-sample', type=int, default=10)
parser.add_argument(
"--snap_individual",
action="store_true",
help="whether to snapshot each individual epoch/iteration")
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument("--fetch_mode", type=int, default=1)
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
# with open(pid_file_path, "w") as file_obj:
# file_obj.write(pid)
# file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
paper_report_label, class_num = squeeze_label_num_report(
args.database, args.use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
use_feature_map_res45 = (args.conv_rnn_type != ConvRNNType.conv_rcnn) and (
args.conv_rnn_type != ConvRNNType.fc_lstm)
use_au_rcnn_loss = (args.conv_rnn_type == ConvRNNType.conv_rcnn)
au_rcnn_train_chain_list = []
if args.backbone == 'vgg':
au_rcnn = AU_RCNN_VGG16(pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
use_roi_align=args.roi_align)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
elif args.backbone == 'resnet101':
if args.two_stream_mode != TwoStreamMode.spatial_temporal:
pretrained_model = args.pretrained_model_rgb if args.pretrained_model_rgb else args.pretrained_model_of
au_rcnn = AU_RCNN_Resnet101(
pretrained_model=pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align,
use_feature_map_res45=use_feature_map_res45,
use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm
or args.conv_rnn_type
== ConvRNNType.sep_conv_lstm),
use_optical_flow_input=(
args.two_stream_mode == TwoStreamMode.optical_flow),
temporal_length=args.sample_frame)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
else:
au_rcnn_rgb = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align,
use_feature_map_res45=use_feature_map_res45,
use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm
or args.conv_rnn_type
== ConvRNNType.sep_conv_lstm),
use_optical_flow_input=False,
temporal_length=args.sample_frame)
au_rcnn_optical_flow = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_of,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align,
use_feature_map_res45=use_feature_map_res45,
use_feature_map_res5=(args.conv_rnn_type != ConvRNNType.fc_lstm
or args.conv_rnn_type
== ConvRNNType.sep_conv_lstm),
use_optical_flow_input=True,
temporal_length=args.sample_frame)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb)
au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor(
au_rcnn_optical_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow)
elif args.backbone == "mobilenet_v1":
au_rcnn = AU_RCNN_MobilenetV1(
pretrained_model_type=args.pretrained_model_args,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
classify_mode=use_au_rcnn_loss,
n_class=class_num,
use_roi_align=args.roi_align)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
if use_au_rcnn_loss:
au_rcnn_train_loss = AU_RCNN_TrainChainLoss()
loss_head_module = au_rcnn_train_loss
elif args.conv_rnn_type == ConvRNNType.conv_lstm:
label_dependency_layer = None
if args.use_label_dependency:
label_dependency_layer = LabelDependencyRNNLayer(
args.database,
in_size=2048,
class_num=class_num,
train_mode=True,
label_win_size=args.label_win_size)
space_time_conv_lstm = SpaceTimeConv(
label_dependency_layer,
args.use_label_dependency,
class_num,
spatial_edge_mode=args.spatial_edge_mode,
temporal_edge_mode=args.temporal_edge_mode,
conv_rnn_type=args.conv_rnn_type)
loss_head_module = space_time_conv_lstm
elif args.conv_rnn_type == ConvRNNType.sep_conv_lstm:
space_time_sep_conv_lstm = SpaceTimeSepConv(
database=args.database,
class_num=class_num,
spatial_edge_mode=args.spatial_edge_mode,
temporal_edge_mode=args.temporal_edge_mode)
loss_head_module = space_time_sep_conv_lstm
elif args.conv_rnn_type == ConvRNNType.fc_lstm:
space_time_fc_lstm = SpaceTimeSepFcLSTM(
database=args.database,
class_num=class_num,
spatial_edge_mode=args.spatial_edge_mode,
temporal_edge_mode=args.temporal_edge_mode)
loss_head_module = space_time_fc_lstm
model = Wrapper(au_rcnn_train_chain_list,
loss_head_module,
args.database,
args.sample_frame,
use_feature_map=use_feature_map_res45,
two_stream_mode=args.two_stream_mode)
batch_size = args.batch_size
img_dataset = AUDataset(database=args.database,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False)
train_video_data = AU_video_dataset(
au_image_dataset=img_dataset,
sample_frame=args.sample_frame,
train_mode=(args.two_stream_mode != TwoStreamMode.optical_flow),
paper_report_label_idx=paper_report_label_idx,
)
Transform = Transform3D
train_video_data = TransformDataset(train_video_data,
Transform(au_rcnn, mirror=False))
if args.proc_num == 1:
train_iter = SerialIterator(train_video_data,
batch_size * args.sample_frame,
repeat=True,
shuffle=False)
else:
train_iter = MultiprocessIterator(train_video_data,
batch_size=batch_size *
args.sample_frame,
n_processes=args.proc_num,
repeat=True,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
if len(args.gpu) > 1:
for gpu in args.gpu:
chainer.cuda.get_device_from_id(gpu).use()
else:
chainer.cuda.get_device_from_id(args.gpu[0]).use()
model.to_gpu(args.gpu[0])
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
key_str = "{0}_fold_{1}".format(args.fold, args.split_idx)
file_list = []
file_list.extend(os.listdir(args.out))
snapshot_model_file_name = args.out + os.sep + filter_last_checkpoint_filename(
file_list, "model", key_str)
# BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz
use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_avail_label"
roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling"
label_dependency_layer_key_str = "label_dep_layer" if args.use_label_dependency else "no_label_dep"
single_model_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}@sampleframe#{10}_model.npz'.format(args.database,
args.fold, args.split_idx,
args.backbone, args.spatial_edge_mode,
args.temporal_edge_mode,
use_paper_key_str, roi_align_key_str,
label_dependency_layer_key_str,
args.conv_rnn_type,args.sample_frame )#, args.label_win_size)
print(single_model_file_name)
pretrained_optimizer_file_name = args.out + os.sep +\
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}@sampleframe#{10}_optimizer.npz'.format(args.database,
args.fold, args.split_idx,
args.backbone, args.spatial_edge_mode,
args.temporal_edge_mode,
use_paper_key_str, roi_align_key_str,
label_dependency_layer_key_str,
args.conv_rnn_type, args.sample_frame)# args.label_win_size)
print(pretrained_optimizer_file_name)
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if args.snap_individual:
if os.path.exists(snapshot_model_file_name) and os.path.isfile(
snapshot_model_file_name):
print("loading pretrained snapshot:{}".format(
snapshot_model_file_name))
chainer.serializers.load_npz(snapshot_model_file_name, model)
else:
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(
single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model)
if args.fix:
au_rcnn = model.au_rcnn_train_chain.au_rcnn
au_rcnn.extractor.conv1.W.update_rule.enabled = False
au_rcnn.extractor.bn1.gamma.update_rule.enabled = False
au_rcnn.extractor.bn1.beta.update_rule.enabled = False
res2_names = ["a", "b1", "b2"]
for res2_name in res2_names:
if res2_name == "a":
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv4.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.beta.update_rule.enabled = False
elif res2_name.startswith("b"):
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
# if (args.spatial_edge_mode in [SpatialEdgeMode.ld_rnn, SpatialEdgeMode.bi_ld_rnn] or args.temporal_edge_mode in \
# [TemporalEdgeMode.ld_rnn, TemporalEdgeMode.bi_ld_rnn]) or (args.conv_rnn_type != ConvRNNType.conv_rcnn):
# updater = BPTTUpdater(train_iter, optimizer, converter=lambda batch, device: concat_examples(batch, device,
# padding=0), device=args.gpu[0])
if len(args.gpu) > 1:
gpu_dict = {"main": args.gpu[0]} # many gpu will use
parallel_models = {"parallel": model.au_rcnn_train_chain}
for slave_gpu in args.gpu[1:]:
gpu_dict[slave_gpu] = int(slave_gpu)
updater = PartialParallelUpdater(
train_iter,
optimizer,
args.database,
models=parallel_models,
devices=gpu_dict,
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
else:
print("only one GPU({0}) updater".format(args.gpu[0]))
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=args.gpu[0],
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
@training.make_extension(trigger=(1, "epoch"))
def reset_order(trainer):
print("reset dataset order after one epoch")
if args.debug:
trainer.updater._iterators[
"main"].dataset._dataset.reset_for_debug_mode()
else:
trainer.updater._iterators[
"main"].dataset._dataset.reset_for_train_mode()
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(reset_order)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
if not args.snap_individual:
trainer.extend(chainer.training.extensions.snapshot_object(
model, filename=os.path.basename(single_model_file_name)),
trigger=(args.snapshot, 'iteration'))
else:
snap_model_file_name = '{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@{7}@{8}@{9}sampleframe#{10}@win#{11}_'.format(
args.database, args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode, use_paper_key_str,
roi_align_key_str, label_dependency_layer_key_str,
args.conv_rnn_type, args.sample_frame, args.label_win_size)
snap_model_file_name = snap_model_file_name + "{.updater.iteration}.npz"
trainer.extend(chainer.training.extensions.snapshot_object(
model, filename=snap_model_file_name),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 10, 'iteration'
plot_interval = 10, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.1, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="log_{0}_fold_{1}_{2}@{3}@{4}@{5}.log".format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type)))
# trainer.reporter.add_observer("main_par", model.loss_head_module)
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/accuracy',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name='loss_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/accuracy'],
file_name='accuracy_{0}_fold_{1}_{2}@{3}@{4}@{5}.png'.format(
args.fold, args.split_idx, args.backbone,
args.spatial_edge_mode, args.temporal_edge_mode,
args.conv_rnn_type),
trigger=plot_interval),
trigger=plot_interval)
trainer.run()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size',
'-b',
type=int,
default=1,
help='each batch size will be a new file')
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='gpu that used to extract feature')
parser.add_argument("--mirror",
action="store_true",
help="whether to mirror")
parser.add_argument("--out_dir",
'-o',
default="/home/machen/dataset/extract_features/")
parser.add_argument(
"--model",
'-m',
help="the AU R-CNN pretrained model file to load to extract feature")
parser.add_argument("--trainval_test", '-tt', help="train or test")
parser.add_argument("--database", default="BP4D")
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask')
parser.add_argument('--proc_num', type=int, default=10)
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument('--mean_rgb',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--mean_flow',
default=config.ROOT_PATH + "BP4D/idx/mean_flow.npy",
help='image mean .npy file')
args = parser.parse_args()
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
return_dict = extract_mode(args.model)
database = return_dict["database"]
fold = return_dict["fold"]
split_idx = return_dict["split_idx"]
backbone = return_dict["backbone"]
use_paper_num_label = return_dict["use_paper_num_label"]
roi_align = return_dict["use_roi_align"]
two_stream_mode = return_dict["two_stream_mode"]
T = return_dict["T"]
class_num = len(config.paper_use_BP4D) if database == "BP4D" else len(
config.paper_use_DISFA)
paper_report_label_idx = sorted(list(config.AU_SQUEEZE.keys()))
if use_paper_num_label:
paper_report_label, class_num = squeeze_label_num_report(
database, True)
paper_report_label_idx = list(paper_report_label.keys())
assert two_stream_mode == TwoStreamMode.rgb_flow
if two_stream_mode == TwoStreamMode.rgb_flow:
au_rcnn_train_chain_list = []
au_rcnn_rgb = AU_RCNN_Resnet101(pretrained_model=backbone,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=roi_align,
use_optical_flow_input=False,
temporal_length=T)
au_rcnn_optical_flow = AU_RCNN_Resnet101(pretrained_model=backbone,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=roi_align,
use_optical_flow_input=True,
temporal_length=T)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb)
au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor(
au_rcnn_optical_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow)
model = Wrapper(au_rcnn_train_chain_list,
class_num,
database,
T,
two_stream_mode=two_stream_mode,
gpus=[args.gpu, args.gpu])
assert os.path.exists(args.model)
print("loading model file : {}".format(args.model))
chainer.serializers.load_npz(args.model, model)
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
if isinstance(model, FasterRCNNResnet101):
model.to_gpu(args.gpu)
img_dataset = AUDataset(database=database,
L=T,
fold=fold,
split_name=args.trainval_test,
split_index=split_idx,
mc_manager=mc_manager,
train_all_data=False,
paper_report_label_idx=paper_report_label_idx,
jump_exists=True,
npz_dir=args.out_dir)
mirror_list = [
False,
]
if args.mirror and args.trainval_test == 'trainval':
mirror_list.append(True)
for mirror in mirror_list:
train_dataset = TransformDataset(
img_dataset,
Transform(T,
mean_rgb_path=args.mean_rgb,
mean_flow_path=args.mean_flow,
mirror=mirror))
if args.proc_num > 1:
dataset_iter = MultiprocessIterator(train_dataset,
batch_size=args.batch_size,
n_processes=args.proc_num,
repeat=False,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
else:
dataset_iter = SerialIterator(train_dataset,
batch_size=args.batch_size,
repeat=False,
shuffle=False)
with chainer.no_backprop_mode(), chainer.using_config(
'cudnn_deterministic',
True), chainer.using_config('train', False):
model_dump = DumpRoIFeature(
dataset_iter,
model,
args.gpu,
database,
converter=lambda batch, device: concat_examples_not_string(
batch, device, padding=0),
output_path=args.out_dir,
trainval_test=args.trainval_test,
fold_split_idx=split_idx,
mirror_data=mirror)
model_dump.evaluate()
def main():
parser = argparse.ArgumentParser(
description='Space Time Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='end_to_end_result',
help='Output directory')
parser.add_argument('--trainval', default='train', help='train/test')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--need_validate',
action='store_true',
help='do or not validate during training')
parser.add_argument('--mean',
default=config.ROOT_PATH +
"BP4D/idx/mean_no_enhance.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="mobilenet_v1",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='RMSprop',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model',
default='mobilenet_v1',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument('--pretrained_model_args',
nargs='+',
type=float,
help='you can pass in "1.0 224" or "0.75 224"')
parser.add_argument('--spatial_edge_mode',
type=SpatialEdgeMode,
choices=list(SpatialEdgeMode),
help='1:all_edge, 2:configure_edge, 3:no_edge')
parser.add_argument(
'--temporal_edge_mode',
type=TemporalEdgeMode,
choices=list(TemporalEdgeMode),
help='1:rnn, 2:attention_block, 3.point-wise feed forward(no temporal)'
)
parser.add_argument("--bi_lstm",
action="store_true",
help="whether to use bi-lstm as Edge/Node RNN")
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--layers", type=int, default=1)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument("--previous_frame", type=int, default=50)
parser.add_argument("--sample_frame", '-sample', type=int, default=25)
parser.add_argument(
"--snap_individual",
action="store_true",
help="whether to snapshot each individual epoch/iteration")
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
train_data = AUDataset(
database=args.database,
fold=args.fold,
split_name=args.trainval,
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False,
)
result_data = [
img_path
for img_path, AU_set, current_database_name in train_data.result_data
if args.database + "|" + img_path not in mc_manager
]
sub_list = split_list(result_data, len(result_data) // 100)
for img_path_lst in sub_list:
with Pool(processes=50) as pool:
input_list = [(img_path, None, None) for img_path in img_path_lst]
result =\
pool.starmap(parallel_landmark_and_conn_component, input_list)
pool.close()
pool.join()
for img_path, AU_box_dict, landmark_dict, box_is_whole_image in result:
key_prefix = args.database + "|"
key = key_prefix + img_path
orig_img = cv2.imread(img_path, cv2.IMREAD_COLOR)
new_face, rect = FaceMaskCropper.dlib_face_crop(
orig_img, landmark_dict)
print("write {}".format(key))
if mc_manager is not None and key not in mc_manager:
save_dict = {
"landmark_dict": landmark_dict,
"AU_box_dict": AU_box_dict,
"crop_rect": rect
}
mc_manager.set(key, save_dict)
def main():
print("chainer cudnn enabled: {}".format(chainer.cuda.cudnn_enabled))
parser = argparse.ArgumentParser(
description='Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
default="0",
help='GPU ID, multiple GPU split by comma, \ '
'Note that BPTT updater do not support multi-GPU')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--seed', '-s', type=int, default=0)
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=20)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--need_validate',
action='store_true',
help='do or not validate during training')
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--feature_model',
default="resnet101",
help="vgg16/vgg19/resnet101 for train")
parser.add_argument('--extract_len', type=int, default=1000)
parser.add_argument('--optimizer',
default='RMSprop',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model',
default='resnet101',
help='imagenet/vggface/resnet101/*.npz')
parser.add_argument('--pretrained_model_args',
nargs='+',
type=float,
help='you can pass in "1.0 224" or "0.75 224"')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument(
"--snap_individual",
action="store_true",
help="whether to snapshot each individual epoch/iteration")
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument(
"--use_sigmoid_cross_entropy",
"-sigmoid",
action="store_true",
help="whether to use sigmoid cross entropy or softmax cross entropy")
parser.add_argument(
"--is_pretrained",
action="store_true",
help="whether is to pretrain BP4D later will for DISFA dataset or not")
parser.add_argument(
"--pretrained_target",
'-pt',
default="",
help="whether pretrain label set will use DISFA or not")
parser.add_argument("--fix",
'-fix',
action="store_true",
help="whether to fix first few conv layers or not")
parser.add_argument(
'--occlude',
default='',
help=
'whether to use occlude face of upper/left/right/lower/none to test')
parser.add_argument("--prefix",
'-prefix',
default="",
help="_beta, for example 3_fold_beta")
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
parser.add_argument("--img_resolution", type=int, default=512)
parser.add_argument(
"--FERA",
action='store_true',
help='whether to use FERA data split train and validate')
parser.add_argument(
'--FPN',
action="store_true",
help=
"whether to use feature pyramid network for training and prediction")
parser.add_argument(
'--fake_box',
action="store_true",
help="whether to use fake average box coordinate to predict")
parser.add_argument('--roi_align',
action="store_true",
help="whether to use roi_align or roi_pooling")
parser.add_argument("--train_test", default="trainval", type=str)
parser.add_argument("--trail_times", default=20, type=int)
parser.add_argument("--each_trail_iteration", default=1000, type=int)
args = parser.parse_args()
if not os.path.exists(args.pid):
os.makedirs(args.pid)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
# with open(pid_file_path, "w") as file_obj:
# file_obj.write(pid)
# file_obj.flush()
config.IMG_SIZE = (args.img_resolution, args.img_resolution)
print('GPU: {}'.format(args.gpu))
if args.is_pretrained:
adaptive_AU_database(args.pretrained_target)
else:
adaptive_AU_database(args.database)
np.random.seed(args.seed)
# 需要先构造一个list的txt文件:id_trainval_0.txt, 每一行是subject + "/" + emotion_seq + "/" frame
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
if args.FPN:
faster_rcnn = FPN101(len(config.AU_SQUEEZE),
pretrained_resnet=args.pretrained_model,
use_roialign=args.roi_align,
mean_path=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution)
elif args.feature_model == 'vgg16':
faster_rcnn = FasterRCNNVGG16(
n_fg_class=len(config.AU_SQUEEZE),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len,
fix=args.fix) # 可改为/home/nco/face_expr/result/snapshot_model.npz
elif args.feature_model == 'vgg19':
faster_rcnn = FasterRCNNVGG19(n_fg_class=len(config.AU_SQUEEZE),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len,
dataset=args.database,
fold=args.fold,
split_idx=args.split_idx)
elif args.feature_model == 'resnet101':
faster_rcnn = FasterRCNNResnet101(
n_fg_class=len(config.AU_SQUEEZE),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len
) # 可改为/home/nco/face_expr/result/snapshot_model.npz
elif args.feature_model == "mobilenet_v1":
faster_rcnn = FasterRCNN_MobilenetV1(
pretrained_model_type=args.pretrained_model_args,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
mean_file=args.mean,
n_class=len(config.AU_SQUEEZE))
batch_size = args.batch_size
with chainer.no_backprop_mode(), chainer.using_config("train", False):
test_data = AUDataset(database=args.database,
fold=args.fold,
img_resolution=args.img_resolution,
split_name=args.train_test,
split_index=args.split_idx,
mc_manager=mc_manager,
train_all_data=False,
prefix=args.prefix,
pretrained_target=args.pretrained_target,
is_FERA=args.FERA)
test_data = TransformDataset(test_data,
Transform(faster_rcnn, mirror=False))
if args.fake_box:
test_data = TransformDataset(test_data,
FakeBoxTransform(args.database))
if args.proc_num == 1:
test_iter = SerialIterator(test_data,
args.batch_size,
repeat=False,
shuffle=True)
else:
test_iter = MultiprocessIterator(test_data,
batch_size=args.batch_size,
n_processes=args.proc_num,
repeat=False,
shuffle=True,
n_prefetch=10,
shared_mem=10000000)
gpu = int(args.gpu) if "," not in args.gpu else int(
args.gpu[:args.gpu.index(",")])
chainer.cuda.get_device_from_id(gpu).use()
faster_rcnn.to_gpu(gpu)
evaluator = SpeedEvaluator(
test_iter,
faster_rcnn,
lambda batch, device: concat_examples_not_none(
batch, device, padding=-99),
device=gpu,
trail_times=args.trail_times,
each_trail_iteration=args.each_trail_iteration,
database=args.database)
observation = evaluator.evaluate()
with open(args.out + os.path.sep + "evaluation_speed_test.json",
"w") as file_obj:
file_obj.write(
json.dumps(observation, indent=4, separators=(',', ': ')))
file_obj.flush()
def main():
parser = argparse.ArgumentParser(
description='Space Time Action Unit R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/SpaceTime_AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
nargs='+',
type=int,
help='GPU ID, multiple GPU split by space')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='output_two_stream',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=1)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--mean_rgb',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--mean_flow',
default=config.ROOT_PATH + "BP4D/idx/mean_flow.npy",
help='image mean .npy file')
parser.add_argument('--backbone',
default="resnet101",
help="vgg/resnet101/mobilenet_v1 for train")
parser.add_argument('--optimizer',
default='SGD',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model_rgb',
help='imagenet/mobilenet_v1/resnet101/*.npz')
parser.add_argument(
'--pretrained_model_flow',
help=
"path of optical flow pretrained model, can also use the same npz with rgb"
)
parser.add_argument('--two_stream_mode',
type=TwoStreamMode,
choices=list(TwoStreamMode),
help='rgb_flow/ optical_flow/ rgb')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--fix",
action="store_true",
help="fix parameter of conv2 update when finetune")
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--use_paper_num_label",
action="store_true",
help="only to use paper reported number of labels"
" to train")
parser.add_argument(
"--roi_align",
action="store_true",
help="whether to use roi align or roi pooling layer in CNN")
parser.add_argument("--T", '-T', type=int, default=10)
parser.add_argument("--proc_num", "-proc", type=int, default=1)
args = parser.parse_args()
os.makedirs(args.pid, exist_ok=True)
os.makedirs(args.out, exist_ok=True)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
print('GPU: {}'.format(",".join(list(map(str, args.gpu)))))
adaptive_AU_database(args.database)
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
paper_report_label, class_num = squeeze_label_num_report(
args.database, args.use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
au_rcnn_train_chain_list = []
if args.backbone == 'vgg':
au_rcnn = AU_RCNN_VGG16(pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
elif args.backbone == 'resnet101':
if args.two_stream_mode != TwoStreamMode.rgb_flow:
assert (args.pretrained_model_rgb == "" and args.pretrained_model_flow != "") or\
(args.pretrained_model_rgb != "" and args.pretrained_model_flow == "")
pretrained_model = args.pretrained_model_rgb if args.pretrained_model_rgb else args.pretrained_model_flow
au_rcnn = AU_RCNN_Resnet101(
pretrained_model=pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align,
use_optical_flow_input=(
args.two_stream_mode == TwoStreamMode.optical_flow),
temporal_length=args.T)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
else: # rgb_flow mode
au_rcnn_rgb = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_rgb,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align,
use_optical_flow_input=False,
temporal_length=args.T)
au_rcnn_optical_flow = AU_RCNN_Resnet101(
pretrained_model=args.pretrained_model_flow,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=args.roi_align,
use_optical_flow_input=True,
temporal_length=args.T)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb)
au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor(
au_rcnn_optical_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow)
model = Wrapper(au_rcnn_train_chain_list,
class_num,
args.database,
args.T,
two_stream_mode=args.two_stream_mode,
gpus=args.gpu)
batch_size = args.batch_size
img_dataset = AUDataset(database=args.database,
L=args.T,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
two_stream_mode=args.two_stream_mode,
train_all_data=False,
paper_report_label_idx=paper_report_label_idx)
train_dataset = TransformDataset(
img_dataset,
Transform(L=args.T,
mirror=True,
mean_rgb_path=args.mean_rgb,
mean_flow_path=args.mean_flow))
if args.proc_num == 1:
train_iter = SerialIterator(train_dataset,
batch_size,
repeat=True,
shuffle=True)
else:
train_iter = MultiprocessIterator(train_dataset,
batch_size=batch_size,
n_processes=args.proc_num,
repeat=True,
shuffle=True,
n_prefetch=3,
shared_mem=53457280)
if len(args.gpu) > 1:
for gpu in args.gpu:
chainer.cuda.get_device_from_id(gpu).use()
else:
chainer.cuda.get_device_from_id(args.gpu[0]).use()
model.to_gpu(args.gpu[0])
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
# BP4D_3_fold_1_resnet101@rnn@no_temporal@use_paper_num_label@roi_align@label_dep_layer@conv_lstm@sampleframe#13_model.npz
use_paper_key_str = "use_paper_num_label" if args.use_paper_num_label else "all_avail_label"
roi_align_key_str = "roi_align" if args.roi_align else "roi_pooling"
single_model_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_model.npz'.format(args.database,
args.fold,
args.split_idx,
args.backbone,
args.two_stream_mode,
use_paper_key_str,
roi_align_key_str,
args.T)
print(single_model_file_name)
pretrained_optimizer_file_name = args.out + os.sep + \
'{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}_optimizer.npz'.format(
args.database,
args.fold, args.split_idx,
args.backbone, args.two_stream_mode,
use_paper_key_str, roi_align_key_str,
args.T)
print(pretrained_optimizer_file_name)
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model)
if args.fix:
au_rcnn = model.au_rcnn_train_chain.au_rcnn
au_rcnn.extractor.conv1.W.update_rule.enabled = False
au_rcnn.extractor.bn1.gamma.update_rule.enabled = False
au_rcnn.extractor.bn1.beta.update_rule.enabled = False
res2_names = ["a", "b1", "b2"]
for res2_name in res2_names:
if res2_name == "a":
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv4.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn4.beta.update_rule.enabled = False
elif res2_name.startswith("b"):
getattr(au_rcnn.extractor.res2,
res2_name).conv1.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn1.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv2.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).conv3.W.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn2.beta.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.gamma.update_rule.enabled = False
getattr(au_rcnn.extractor.res2,
res2_name).bn3.beta.update_rule.enabled = False
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=args.gpu[0],
converter=lambda batch, device: concat_examples(
batch, device, padding=0))
trainer = training.Trainer(updater, (10, 'iteration'), out=args.out)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
trainer.extend(chainer.training.extensions.snapshot_object(
model, filename=os.path.basename(single_model_file_name)),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 100, 'iteration'
plot_interval = 10, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.1, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="log_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.log".format(
args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T)))
# trainer.reporter.add_observer("main_par", model.loss_head_module)
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/accuracy',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name="loss_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.png".format(
args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/accuracy'],
file_name="accuracy_{0}_{1}_fold_{2}_{3}@{4}@{5}@{6}@T#{7}.png".
format(args.database, args.fold, args.split_idx, args.backbone,
args.two_stream_mode, use_paper_key_str, roi_align_key_str,
args.T),
trigger=plot_interval),
trigger=plot_interval)
# trainer.run()
cProfile.runctx("trainer.run()", globals(), locals(), "Profile.prof")
s = pstats.Stats("Profile.prof")
s.strip_dirs().sort_stats("time").print_stats()
for file_path in file_path_list:
print("processing {}".format(file_path))
with open(file_path, "r") as file_obj:
for line in file_obj:
path = line.split()[0]
print("processing {}".format(path))
database = line.split()[-1]
abs_path = config.RGB_PATH[database] + "/" + path
AU_group_box_area = stats_AU_group_area(
abs_path, mc_cached, database)
for AU_couple, area in AU_group_box_area.items():
all_AU_group[AU_couple].append(area)
for AU_couple, area_list in all_AU_group.items():
print(AU_couple, sum(area_list) / len(area_list))
if __name__ == "__main__":
database = "BP4D"
file_path_list = [
"/home/machen/dataset/{}/idx/3_fold/id_trainval_1.txt".format(
database),
"/home/machen/dataset/{}/idx/3_fold/id_test_1.txt".format(database)
]
from collections_toolkit.memcached_manager import PyLibmcManager
adaptive_AU_database(database)
mc_manager = PyLibmcManager('127.0.0.1')
read_idx_file(file_path_list, mc_manager)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu',
'-g',
type=int,
default=0,
help='GPU ID (negative value indicates CPU)'
) # open_crf layer only works for CPU mode
parser.add_argument(
"--model", "-m",
help="pretrained model file path") # which contains pretrained target
parser.add_argument("--pretrained_model", "-pre", default="resnet101")
parser.add_argument("--memcached_host", default="127.0.0.1")
parser.add_argument('--mean_rgb',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--mean_flow',
default=config.ROOT_PATH + "BP4D/idx/mean_flow.npy",
help='image mean .npy file')
parser.add_argument('--proc_num',
type=int,
default=10,
help="multiprocess fetch data process number")
parser.add_argument('--batch',
'-b',
type=int,
default=10,
help='mini batch size')
args = parser.parse_args()
if not args.model.endswith("model.npz"):
return
model_info = extract_mode(args.model)
database = model_info["database"]
fold = model_info["fold"]
split_idx = model_info["split_idx"]
backbone = model_info["backbone"]
use_paper_num_label = model_info["use_paper_num_label"]
use_roi_align = model_info["use_roi_align"]
two_stream_mode = model_info['two_stream_mode']
T = model_info["T"]
adaptive_AU_database(database)
paper_report_label, class_num = squeeze_label_num_report(
database, use_paper_num_label)
paper_report_label_idx = list(paper_report_label.keys())
if not paper_report_label_idx:
paper_report_label_idx = None
class_num = len(config.AU_SQUEEZE)
else:
class_num = len(paper_report_label_idx)
model_print_dict = OrderedDict()
for key, value in model_info.items():
model_print_dict[key] = str(value)
print("""
{0}
======================================
INFO:
{1}
======================================
""".format(args.model,
json.dumps(model_print_dict, sort_keys=True, indent=8)))
au_rcnn_train_chain_list = []
if backbone == 'resnet101':
if two_stream_mode != TwoStreamMode.rgb_flow:
pretrained_model = backbone
au_rcnn = AU_RCNN_Resnet101(
pretrained_model=pretrained_model,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=use_roi_align,
use_optical_flow_input=(
two_stream_mode == TwoStreamMode.optical_flow),
temporal_length=T)
au_rcnn_train_chain = AU_RCNN_ROI_Extractor(au_rcnn)
au_rcnn_train_chain_list.append(au_rcnn_train_chain)
else: # rgb_flow mode
au_rcnn_rgb = AU_RCNN_Resnet101(pretrained_model=backbone,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=use_roi_align,
use_optical_flow_input=False,
temporal_length=T)
au_rcnn_optical_flow = AU_RCNN_Resnet101(
pretrained_model=backbone,
min_size=config.IMG_SIZE[0],
max_size=config.IMG_SIZE[1],
use_roi_align=use_roi_align,
use_optical_flow_input=True,
temporal_length=T)
au_rcnn_train_chain_rgb = AU_RCNN_ROI_Extractor(au_rcnn_rgb)
au_rcnn_train_chain_optical_flow = AU_RCNN_ROI_Extractor(
au_rcnn_optical_flow)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_rgb)
au_rcnn_train_chain_list.append(au_rcnn_train_chain_optical_flow)
au_rcnn = au_rcnn_rgb
model = Wrapper(au_rcnn_train_chain_list,
class_num,
database,
T,
two_stream_mode=two_stream_mode,
gpus=[args.gpu, args.gpu])
chainer.serializers.load_npz(args.model, model)
print("loading {}".format(args.model))
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
mc_manager = PyLibmcManager(args.memcached_host)
img_dataset = AUDataset(database=database,
L=T,
fold=fold,
split_name='test',
split_index=split_idx,
mc_manager=mc_manager,
train_all_data=False,
two_stream_mode=two_stream_mode,
paper_report_label_idx=paper_report_label_idx)
video_dataset = TransformDataset(
img_dataset,
Transform(L=T,
mean_rgb_path=args.mean_rgb,
mean_flow_path=args.mean_flow))
if args.proc_num == 1:
test_iter = SerialIterator(video_dataset,
batch_size=args.batch,
repeat=False,
shuffle=False)
else:
test_iter = MultiprocessIterator(video_dataset,
batch_size=args.batch,
n_processes=args.proc_num,
repeat=False,
shuffle=False,
n_prefetch=10,
shared_mem=10000000)
with chainer.no_backprop_mode(), chainer.using_config(
'cudnn_deterministic', True), chainer.using_config('train', False):
predict_data_path = os.path.dirname(
args.model) + os.path.sep + "pred_" + os.path.basename(
args.model)[:os.path.basename(args.model).rindex("_")] + ".npz"
print("npz_path: {}".format(predict_data_path))
au_evaluator = ActionUnitEvaluator(
test_iter,
model,
args.gpu,
database=database,
paper_report_label=paper_report_label,
converter=lambda batch, device: concat_examples_not_labels(
batch, device, padding=0),
T=T,
output_path=predict_data_path)
observation = au_evaluator.evaluate()
with open(os.path.dirname(args.model) + os.path.sep + "evaluation_result_{0}.json".format(os.path.basename(args.model)\
[:os.path.basename(args.model).rindex("_")]
), "w") as file_obj:
file_obj.write(
json.dumps(observation, indent=4, separators=(',', ': ')))
file_obj.flush()