def __init__(self, config, loss_for_G, graph):
""" Build up the graph
Inputs
------
config :
* batch_size : mini batch size
* log_dir : path to save training summary
* learning_rate : adam's learning rate
* seq_length : length of sequence during training
* latent_dims : latent dimensions
* penalty_lambda = gradient penalty's weight, ref from paper 'improved-wgan'
* n_resblock : number of resblock in network body
* if_feed_extra_info : basket position
* residual_alpha : residual block = F(x) * residual_alpha + x
* leaky_relu_alpha : tf.maximum(x, leaky_relu_alpha * x)
* n_filters : number of filters in all ConV
loss_for_G : function
from Critic Network, given generative fake result, scores in return
graph :
tensorflow default graph
"""
self.data_factory = DataFactory()
# hyper-parameters
self.batch_size = config.batch_size
self.log_dir = config.log_dir
self.learning_rate = config.learning_rate
self.seq_length = config.seq_length
self.latent_dims = config.latent_dims
self.penalty_lambda = config.penalty_lambda
self.latent_penalty_lambda = config.latent_penalty_lambda
self.n_resblock = config.n_resblock
self.if_feed_extra_info = config.if_feed_extra_info
self.residual_alpha = config.residual_alpha
self.leaky_relu_alpha = config.leaky_relu_alpha
self.n_filters = config.n_filters
# steps
self.__global_steps = tf.train.get_or_create_global_step(graph=graph)
with tf.name_scope('Generator'):
self.__steps = tf.get_variable('G_steps', shape=[
], dtype=tf.int32, initializer=tf.zeros_initializer(dtype=tf.int32), trainable=False)
# IO
self.loss_for_G = loss_for_G
self.__z = tf.placeholder(dtype=tf.float32, shape=[
None, self.latent_dims], name='latent_input')
self.__cond = tf.placeholder(dtype=tf.float32, shape=[
None, None, 13], name='team_a')
self.__real_data = tf.placeholder(dtype=tf.float32, shape=[
None, None, 10], name='real_data')
# adversarial learning : wgan
self.__build_model()
# summary
self.__summary_op = tf.summary.merge(tf.get_collection('G'))
self.__summary_histogram_op = tf.summary.merge(
tf.get_collection('G_histogram'))
self.__summary_weight_op = tf.summary.merge(
tf.get_collection('G_weight'))
self.summary_writer = tf.summary.FileWriter(
self.log_dir + 'G', graph=graph)
def __init__(self, config):
self.global_step = tf.train.get_or_create_global_step()
self.lr_ = config.lr_
self.batch_size = config.batch_size
self.seq_length = config.seq_length
self.latent_dims = config.latent_dims
self.n_filters = config.n_filters
self.features_ = config.features_
self.features_d = config.features_d
self.keep_prob = config.keep_prob
self.n_resblock = config.n_resblock
self.data_factory = DataFactory()
#Real offence
self.input_ = tf.placeholder(tf.float32,
shape=[None, None, self.features_],
name='Real')
#Real Defence
self.input_d = tf.placeholder(tf.float32,
shape=[None, None, self.features_d],
name='Real_defence')
self.ground_feature = tf.placeholder(tf.float32,
shape=[None, None, 6],
name='Real_feat')
#Condition Data
self.seq_input = tf.placeholder(tf.float32,
shape=[None, None, self.features_],
name='Cond_input')
self.seq_feature = tf.placeholder(tf.float32,
shape=[None, None, 6],
name='Seq_feat')
self.z_sample = tf.placeholder(tf.float32,
shape=[None, self.latent_dims],
name='Latent')
self.network_()
self.loss_()
init_ = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init_)
self.saver = tf.train.Saver(max_to_keep=0)
# summary collection
self.G_summaries = tf.summary.merge(tf.get_collection('G'))
self.D_summaries = tf.summary.merge(tf.get_collection('D'))
# summary writer
self.G_summary_writer = tf.summary.FileWriter(
os.path.join(config.folder_path, 'Log/G'),
graph=tf.get_default_graph())
self.D_summary_writer = tf.summary.FileWriter(
os.path.join(config.folder_path, 'Log/D'))
self.D_valid_summary_writer = tf.summary.FileWriter(
os.path.join(config.folder_path, 'Log/D_valid'))
def __init__(self, config, graph):
""" TO build up the graph
Inputs
------
config :
* batch_size : mini batch size
* log_dir : path to save training summary
* learning_rate : adam's learning rate
* hidden_size : number of hidden units in LSTM
* rnn_layers : number of stacked LSTM
* seq_length : length of LSTM
* latent_dims : dimensions of latent feature
* penalty_lambda = gradient penalty's weight, ref from paper of 'improved-wgan'
graph :
tensorflow default graph
"""
self.data_factory = DataFactory()
# hyper-parameters
self.batch_size = config.batch_size
self.log_dir = config.log_dir
self.learning_rate = config.learning_rate
self.hidden_size = config.hidden_size
self.rnn_layers = config.rnn_layers
self.seq_length = config.seq_length
self.latent_dims = config.latent_dims
self.penalty_lambda = config.penalty_lambda
self.if_log_histogram = config.if_log_histogram
self.n_resblock = config.n_resblock
# steps
self.__global_steps = tf.train.get_or_create_global_step(graph=graph)
self.__steps = 0
# data
self.__G_samples = tf.placeholder(
dtype=tf.float32,
shape=[self.batch_size, self.seq_length, 10],
name='G_samples')
self.__X = tf.placeholder(dtype=tf.float32,
shape=[self.batch_size, self.seq_length, 10],
name='real_data')
# TODO mismatched conditional constraints
# self.__mismatched_cond = tf.placeholder(dtype=tf.float32, shape=[
# self.batch_size, self.seq_length, self.num_features], name='mismatched_cond)
self.__matched_cond = tf.placeholder(
dtype=tf.float32,
shape=[self.batch_size, self.seq_length, 13],
name='matched_cond')
# adversarial learning : wgan
self.__build_wgan()
# summary
self.__summary_op = tf.summary.merge(tf.get_collection('C'))
self.__summary_valid_op = tf.summary.merge(
tf.get_collection('C_valid'))
self.summary_writer = tf.summary.FileWriter(self.log_dir + 'C')
self.valid_summary_writer = tf.summary.FileWriter(self.log_dir +
'C_valid')
def __init__(self, config, critic_inference, graph):
""" TO build up the graph
Inputs
------
config :
* batch_size : mini batch size
* log_dir : path to save training summary
* learning_rate : adam's learning rate
* seq_length : length of LSTM
* latent_dims : dimensions of latent feature
* penalty_lambda = gradient penalty's weight, ref from paper of 'improved-wgan'
graph :
tensorflow default graph
"""
self.data_factory = DataFactory()
# hyper-parameters
self.batch_size = config.batch_size
self.log_dir = config.log_dir
self.learning_rate = config.learning_rate
self.seq_length = config.seq_length
self.latent_dims = config.latent_dims
self.num_layers = config.num_layers
self.hidden_size = config.hidden_size
self.num_features = config.num_features
self.if_feed_prev = True
self.penalty_lambda = config.penalty_lambda
self.latent_penalty_lambda = config.latent_penalty_lambda
self.n_resblock = config.n_resblock
self.if_feed_extra_info = config.if_feed_extra_info
self.residual_alpha = config.residual_alpha
self.leaky_relu_alpha = config.leaky_relu_alpha
# steps
self.__global_steps = tf.train.get_or_create_global_step(graph=graph)
self.__steps = 0
# IO
self.critic = critic_inference
self.__z = tf.placeholder(dtype=tf.float32,
shape=[self.batch_size, self.latent_dims],
name='latent_input')
self.__cond = tf.placeholder(dtype=tf.float32,
shape=[self.batch_size, None, 13],
name='team_a')
# adversarial learning : wgan
self.__build_model()
# summary
self.__summary_op = tf.summary.merge(tf.get_collection('G'))
self.__summary_histogram_op = tf.summary.merge(
tf.get_collection('G_histogram'))
# self.__summary_weight_op = tf.summary.merge(
# tf.get_collection('G_weight'))
self.summary_writer = tf.summary.FileWriter(self.log_dir + 'G',
graph=graph)
def mode_8(sess, graph, save_path):
""" to find high-openshot-penalty data in 1000 real data
"""
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
data_factory = DataFactory(real_data)
train_data, valid_data = data_factory.fetch_data()
# placeholder tensor
real_data_t = graph.get_tensor_by_name('real_data:0')
matched_cond_t = graph.get_tensor_by_name('matched_cond:0')
# result tensor
heuristic_penalty_pframe = graph.get_tensor_by_name(
'Critic/C_inference/heuristic_penalty/Min:0')
# 'Generator/G_loss/C_inference/linear_result/Reshape:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
real_hp_pframe_all = []
for batch_id in range(train_data['A'].shape[0] // FLAGS.batch_size):
index_id = batch_id * FLAGS.batch_size
real_data = train_data['B'][index_id:index_id + FLAGS.batch_size]
cond_data = train_data['A'][index_id:index_id + FLAGS.batch_size]
# real
feed_dict = {real_data_t: real_data, matched_cond_t: cond_data}
real_hp_pframe = sess.run(heuristic_penalty_pframe,
feed_dict=feed_dict)
real_hp_pframe_all.append(real_hp_pframe)
real_hp_pframe_all = np.concatenate(real_hp_pframe_all, axis=0)
print(real_hp_pframe_all.shape)
real_hp_pdata = np.mean(real_hp_pframe_all, axis=1)
mean_ = np.mean(real_hp_pdata)
std_ = np.std(real_hp_pdata)
print(mean_)
print(std_)
concat_AB = np.concatenate([train_data['A'], train_data['B']], axis=-1)
recoverd = data_factory.recover_data(concat_AB)
for i, v in enumerate(real_hp_pdata):
if v > (mean_ + 2 * std_):
print('bad', i, v)
game_visualizer.plot_data(recoverd[i],
recoverd.shape[1],
file_path=save_path + 'bad_' + str(i) +
'_' + str(v) + '.mp4',
if_save=True)
if v < 0.0025:
print('good', i, v)
game_visualizer.plot_data(recoverd[i],
recoverd.shape[1],
file_path=save_path + 'good_' + str(i) +
'_' + str(v) + '.mp4',
if_save=True)
print('!!Completely Saved!!')
def generate_defensive_strategy(sess, graph, offense_input):
""" Given one offensive input, generate 100 defensive strategies, and reture only one result with the hightest score .
Inputs
------
offense_input : float, shape=[lenght,13]
lenght could be variable, [13] -> [ball's xyz * 1, offensive player's xy * 5]
Returns
-------
defense_result : float, shape=[length,10]
lenght could be variable, [10] -> [defensive player's xy * 5]
"""
# placeholder tensor
latent_input_t = graph.get_tensor_by_name('Generator/latent_input:0')
team_a_t = graph.get_tensor_by_name('Generator/team_a:0')
G_samples_t = graph.get_tensor_by_name('Critic/G_samples:0')
matched_cond_t = graph.get_tensor_by_name('Critic/matched_cond:0')
# result tensor
result_t = graph.get_tensor_by_name(
'Generator/G_inference/conv_result/conv1d/Maximum:0')
critic_scores_t = graph.get_tensor_by_name(
'Critic/C_inference_1/conv_output/Reshape:0')
real_data = np.load('../../data/FEATURES-4.npy')
# DataFactory
data_factory = DataFactory(real_data)
conditions = data_factory.normalize_offense(
np.expand_dims(offense_input, axis=0))
# given 100 latents generate 100 results on same condition at once
conditions_duplicated = np.concatenate(
[conditions for _ in range(100)], axis=0)
# generate result
latents = np.random.normal(
0., 1., size=[100, 100])
feed_dict = {
latent_input_t: latents,
team_a_t: conditions_duplicated
}
result = sess.run(
result_t, feed_dict=feed_dict)
# calculate em distance
feed_dict = {
G_samples_t: result,
matched_cond_t: conditions_duplicated
}
critic_scores = sess.run(
critic_scores_t, feed_dict=feed_dict)
recoverd_A_fake_B = data_factory.recover_B(result)
return recoverd_A_fake_B[np.argmax(critic_scores)]
def test():
"""
test only
"""
train_data = np.load(opt.data_path)
data_factory = DataFactory(train_data)
train_data = data_factory.fetch_ori_data()
train_data = data_factory.recover_data(train_data)
for i in range(opt.amount):
plot_data(results_data[i:i + 1],
length=100,
file_path=opt.save_path + 'play_' + str(i) + '.mp4',
if_save=opt.save)
def mode_7(sess, graph, save_path):
""" to draw feature map
"""
# normalize
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
data_factory = DataFactory(real_data)
target_data = np.load('FEATURES-6.npy')[:6]
team_AB = np.concatenate(
[
# ball
target_data[:, :, 0, :3].reshape(
[target_data.shape[0], target_data.shape[1], 1 * 3]),
# team A players
target_data[:, :, 1:6, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2]),
# team B players
target_data[:, :, 6:11, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2])
], axis=-1
)
dummy_AB = np.zeros(shape=[128 - 6, 100, 23])
team_AB = np.concatenate([team_AB, dummy_AB], axis=0)
team_AB = data_factory.normalize(team_AB)
team_A = team_AB[:, :, :13]
team_B = team_AB[:, :, 13:]
# placeholder tensor
latent_input_t = graph.get_tensor_by_name('latent_input:0')
team_a_t = graph.get_tensor_by_name('team_a:0')
# result tensor
conds_linear_t = graph.get_tensor_by_name(
'Generator/G_inference/conds_linear/BiasAdd:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
# result collector
latents = np.concatenate([z_samples(1)
for i in range(FLAGS.batch_size)], axis=0)
feed_dict = {
latent_input_t: latents,
team_a_t: team_A
}
conds_linear = sess.run(conds_linear_t, feed_dict=feed_dict)
for i in range(6):
trace = go.Heatmap(z=conds_linear[i])
data = [trace]
plotly.offline.plot(data, filename=os.path.join(
save_path, 'G_conds_linear' + str(i) + '.html'))
print('!!Completely Saved!!')
def main(_):
with tf.get_default_graph().as_default() as graph:
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
# normalize
data_factory = DataFactory(real_data)
train_data, valid_data = data_factory.fetch_data()
print(train_data['A'].shape)
print(valid_data['A'].shape)
# config setting
config = TrainingConfig()
config.show()
# train
training(train_data, valid_data, data_factory, config, graph)
def main(args):
torch.cuda.set_device(0)
torch.backends.cudnn.benchmark = False
num_anno = torch.tensor(
HICODet(None,
anno_file=os.path.join(
args.data_root,
'instances_train2015.json')).anno_interaction)
rare = torch.nonzero(num_anno < 10).squeeze(1)
non_rare = torch.nonzero(num_anno >= 10).squeeze(1)
dataloader = DataLoader(dataset=DataFactory(
name='hicodet',
partition=args.partition,
data_root=args.data_root,
detection_root=args.detection_dir,
box_score_thresh_h=args.human_thresh,
box_score_thresh_o=args.object_thresh),
collate_fn=custom_collate,
batch_size=1,
num_workers=args.num_workers,
pin_memory=True)
net = SpatioAttentiveGraph(dataloader.dataset.dataset.object_to_verb,
49,
num_iterations=args.num_iter,
max_human=args.max_human,
max_object=args.max_object)
epoch = 0
if os.path.exists(args.model_path):
print("Loading model from ", args.model_path)
checkpoint = torch.load(args.model_path, map_location="cpu")
net.load_state_dict(checkpoint['model_state_dict'])
epoch = checkpoint["epoch"]
elif len(args.model_path):
print("\nWARNING: The given model path does not exist. "
"Proceed to use a randomly initialised model.\n")
net.cuda()
timer = pocket.utils.HandyTimer(maxlen=1)
with timer:
test_ap = test(net, dataloader)
print("Model at epoch: {} | time elapsed: {:.2f}s\n"
"Full: {:.4f}, rare: {:.4f}, non-rare: {:.4f}".format(
epoch, timer[0], test_ap.mean(), test_ap[rare].mean(),
test_ap[non_rare].mean()))
def mode_6(sess, graph, save_path):
""" to draw different length result
"""
# normalize
real_data = np.load(FLAGS.data_path)
print('real_data.shape', real_data.shape)
data_factory = DataFactory(real_data)
target_data = np.load('FEATURES-7.npy')[:, :]
team_AB = np.concatenate(
[
# ball
target_data[:, :, 0, :3].reshape(
[target_data.shape[0], target_data.shape[1], 1 * 3]),
# team A players
target_data[:, :, 1:6, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2]),
# team B players
target_data[:, :, 6:11, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2])
], axis=-1
)
team_AB = data_factory.normalize(team_AB)
team_A = team_AB[:, :, :13]
team_B = team_AB[:, :, 13:]
# placeholder tensor
latent_input_t = graph.get_tensor_by_name('latent_input:0')
team_a_t = graph.get_tensor_by_name('team_a:0')
# result tensor
result_t = graph.get_tensor_by_name(
'Generator/G_inference/conv_result/conv1d/Maximum:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
# result collector
latents = z_samples(team_AB.shape[0])
feed_dict = {
latent_input_t: latents,
team_a_t: team_A
}
result_fake_B = sess.run(result_t, feed_dict=feed_dict)
results_A_fake_B = np.concatenate([team_A, result_fake_B], axis=-1)
results_A_fake_B = data_factory.recover_data(results_A_fake_B)
for i in range(results_A_fake_B.shape[0]):
game_visualizer.plot_data(
results_A_fake_B[i], target_data.shape[1], file_path=save_path + str(i) + '.mp4', if_save=True)
print('!!Completely Saved!!')
def main(args):
torch.cuda.set_device(0)
torch.backends.cudnn.benchmark = False
if not os.path.exists(args.cache_dir):
os.makedirs(args.cache_dir)
dataloader = DataLoader(
dataset=DataFactory(
name=args.dataset, partition=args.partition,
data_root=args.data_root,
detection_root=args.detection_dir,
), collate_fn=custom_collate, batch_size=1,
num_workers=args.num_workers, pin_memory=True
)
if args.dataset == 'hicodet':
object_to_target = dataloader.dataset.dataset.object_to_verb
human_idx = 49
num_classes = 117
elif args.dataset == 'vcoco':
object_to_target = dataloader.dataset.dataset.object_to_action
human_idx = 1
num_classes = 24
net = SCG(
object_to_target, human_idx, num_classes=num_classes,
num_iterations=args.num_iter,
max_human=args.max_human, max_object=args.max_object,
box_score_thresh=args.box_score_thresh
)
if os.path.exists(args.model_path):
print("Loading model from ", args.model_path)
checkpoint = torch.load(args.model_path, map_location="cpu")
net.load_state_dict(checkpoint['model_state_dict'])
elif len(args.model_path):
print("\nWARNING: The given model path does not exist. "
"Proceed to use a randomly initialised model.\n")
net.cuda()
if args.dataset == 'hicodet':
with open(os.path.join(args.data_root, 'coco80tohico80.json'), 'r') as f:
coco2hico = json.load(f)
inference_hicodet(net, dataloader, coco2hico, args.cache_dir)
elif args.dataset == 'vcoco':
inference_vcoco(net, dataloader, args.cache_dir)
def main(_):
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
# normalize
data_factory = DataFactory(real_data)
train_data, valid_data = data_factory.fetch_data()
print(train_data['A'].shape)
print(valid_data['A'].shape)
# config setting
config = TrainingConfig()
baseline_graph = None
if FLAGS.baseline_checkpoint is not None:
baseline_graph = tf.Graph()
default_graph = tf.Graph()
training(train_data, valid_data, data_factory, config, default_graph,
baseline_graph)
def main(args):
dataset = DataFactory(
name='hicodet',
partition=args.partition,
data_root=args.data_root,
detection_root=args.detection_dir,
)
dataloader = torch.utils.data.DataLoader(dataset,
collate_fn=custom_collate,
batch_size=4,
shuffle=False)
net = SCG(dataset.dataset.object_to_verb,
49,
num_iterations=args.num_iter,
box_score_thresh=args.box_score_thresh)
net.eval()
if os.path.exists(args.model_path):
print("\nLoading model from ", args.model_path)
checkpoint = torch.load(args.model_path, map_location="cpu")
net.load_state_dict(checkpoint['model_state_dict'])
elif len(args.model_path):
print("\nWARNING: The given model path does not exist. "
"Proceed to use a randomly initialised model.\n")
else:
print("\nProceed with a randomly initialised model\n")
# iterator = iter(dataloader)
# image, detection, target = next(iterator)
image, detection, target = dataset[args.index]
image = [image]
detection = [detection]
target = [target]
output = net(image, detection, target)
visualise_entire_image(dataset, output[0])
def main(_):
with tf.get_default_graph().as_default() as graph:
real_data = np.load(os.path.join(
FLAGS.data_path, '50Real.npy'))[:, :FLAGS.seq_length, :, :]
seq_data = np.load(os.path.join(FLAGS.data_path, '50Seq.npy'))
features_ = np.load(os.path.join(FLAGS.data_path, 'SeqCond.npy'))
real_feat = np.load(os.path.join(FLAGS.data_path, 'RealCond.npy'))
print("Real Data: ", real_data.shape)
print("Seq Data: ", seq_data.shape)
print("Real Feat: ", real_feat.shape)
print("Seq Feat: ", features_.shape)
data_factory = DataFactory(real_data=real_data,
seq_data=seq_data,
features_=features_,
real_feat=real_feat)
config = Training_config()
config.show()
trainer = Trainer(data_factory, config)
trainer()
def main(args):
torch.cuda.set_device(0)
torch.manual_seed(args.random_seed)
torch.backends.cudnn.benchmark = False
train_loader = DataLoader(dataset=DataFactory(
name=args.dataset,
partition=args.partitions[0],
data_root=args.data_root,
detection_root=args.train_detection_dir,
box_score_thresh_h=args.human_thresh,
box_score_thresh_o=args.object_thresh,
flip=True),
collate_fn=custom_collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
shuffle=True)
val_loader = DataLoader(dataset=DataFactory(
name=args.dataset,
partition=args.partitions[1],
data_root=args.data_root,
detection_root=args.val_detection_dir,
box_score_thresh_h=args.human_thresh,
box_score_thresh_o=args.object_thresh),
collate_fn=custom_collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True)
if args.dataset == 'hicodet':
object_to_target = train_loader.dataset.dataset.object_to_verb
human_idx = 49
num_classes = 117
elif args.dataset == 'vcoco':
object_to_target = train_loader.dataset.dataset.object_to_action
human_idx = 1
num_classes = 24
net = SpatioAttentiveGraph(object_to_target,
human_idx,
num_iterations=args.num_iter,
postprocess=False,
max_human=args.max_human,
max_object=args.max_object)
# Fix backbone parameters
for p in net.backbone.parameters():
p.requires_grad = False
if os.path.exists(args.checkpoint_path):
print("Continue from saved checkpoint ", args.checkpoint_path)
checkpoint = torch.load(args.checkpoint_path, map_location='cpu')
net.load_state_dict(checkpoint['model_state_dict'])
optim_state_dict = checkpoint['optim_state_dict']
sched_state_dict = checkpoint['scheduler_state_dict']
epoch = checkpoint['epoch']
iteration = checkpoint['iteration']
else:
print("Start from a randomly intialised model")
optim_state_dict = None
sched_state_dict = None
epoch = 0
iteration = 0
engine = CustomisedLE(net,
train_loader,
val_loader,
num_classes=num_classes,
optim_params={
'lr': args.learning_rate,
'momentum': args.momentum,
'weight_decay': args.weight_decay
},
optim_state_dict=optim_state_dict,
lr_scheduler=True,
lr_sched_params={
'milestones': args.milestones,
'gamma': args.lr_decay
},
print_interval=args.print_interval,
cache_dir=args.cache_dir)
engine.update_state_key(epoch=epoch, iteration=iteration)
if sched_state_dict is not None:
engine.fetch_state_key('lr_scheduler').load_state_dict(
sched_state_dict)
engine(args.num_epochs)
def mode_5(sess, graph, save_path):
""" to calculate hueristic score on selected result
"""
NORMAL_C_ID = [154, 108, 32, 498, 2, 513, 263, 29, 439, 249, 504, 529, 24, 964, 641, 739, 214, 139, 819, 1078, 772, 349, 676, 1016, 582, 678, 39, 279,
918, 477, 809, 505, 896, 600, 564, 50, 810, 1132, 683, 578, 1131, 887, 621, 1097, 665, 528, 310, 631, 1102, 6, 945, 1020, 853, 490, 64, 1002, 656]
NORMAL_N_ID = [58, 5, 47, 66, 79, 21, 70, 54, 3, 59, 67, 59, 84, 38, 71, 62, 55, 86, 14, 83, 94, 97, 83, 27, 38, 68, 95,
26, 60, 2, 54, 46, 34, 75, 38, 4, 59, 87, 52, 44, 92, 28, 86, 71, 24, 28, 13, 70, 87, 44, 52, 25, 59, 61, 86, 16, 98]
GOOD_C_ID = [976, 879, 293, 750, 908, 878, 831, 1038, 486, 268,
265, 252, 1143, 383, 956, 974, 199, 777, 585, 34, 932]
GOOD_N_ID = [52, 16, 87, 43, 45, 66, 22, 77, 36,
50, 47, 9, 34, 9, 82, 42, 65, 43, 7, 29, 62]
BEST_C_ID = [570, 517, 962, 1088, 35, 623, 1081, 33, 255, 571,
333, 990, 632, 431, 453, 196, 991, 267, 591, 902, 597, 646]
BEST_N_ID = [22, 42, 76, 92, 12, 74, 92, 58, 69, 69,
23, 63, 89, 7, 74, 27, 12, 20, 35, 77, 62, 63]
DUMMY_ID = np.zeros(shape=[28])
ALL_C_ID = np.concatenate(
[NORMAL_C_ID, GOOD_C_ID, BEST_C_ID, DUMMY_ID]).astype(np.int32)
ALL_N_ID = np.concatenate(
[NORMAL_N_ID, GOOD_N_ID, BEST_N_ID, DUMMY_ID]).astype(np.int32)
print(ALL_C_ID.shape)
print(ALL_N_ID.shape)
fake_result_AB = np.load(
'v3/2/collect/mode_1/results_A_fake_B.npy')[ALL_N_ID, ALL_C_ID]
real_result_AB = np.load(
'v3/2/collect/mode_1/results_A_real_B.npy')[ALL_C_ID]
print(fake_result_AB.shape)
print(real_result_AB.shape)
# normalize
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
data_factory = DataFactory(real_data)
fake_result_AB = data_factory.normalize(fake_result_AB)
real_result_AB = data_factory.normalize(real_result_AB)
# placeholder tensor
real_data_t = graph.get_tensor_by_name('real_data:0')
matched_cond_t = graph.get_tensor_by_name('matched_cond:0')
# result tensor
heuristic_penalty_pframe = graph.get_tensor_by_name(
'Critic/C_inference/heuristic_penalty/Min:0')
# 'Generator/G_loss/C_inference/linear_result/Reshape:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
# real
feed_dict = {
real_data_t: real_result_AB[:, :, 13:23],
matched_cond_t: real_result_AB[:, :, :13]
}
real_hp_pframe = sess.run(heuristic_penalty_pframe, feed_dict=feed_dict)
# fake
feed_dict = {
real_data_t: fake_result_AB[:, :, 13:23],
matched_cond_t: fake_result_AB[:, :, :13]
}
fake_hp_pframe = sess.run(heuristic_penalty_pframe, feed_dict=feed_dict)
print(np.mean(real_hp_pframe[:100]))
print(np.mean(fake_hp_pframe[:100]))
print('!!Completely Saved!!')
def main(rank, args):
dist.init_process_group(backend="nccl",
init_method="env://",
world_size=args.world_size,
rank=rank)
trainset = DataFactory(name=args.dataset,
partition=args.partitions[0],
data_root=args.data_root,
detection_root=args.train_detection_dir,
flip=True)
valset = DataFactory(name=args.dataset,
partition=args.partitions[1],
data_root=args.data_root,
detection_root=args.val_detection_dir)
train_loader = DataLoader(dataset=trainset,
collate_fn=custom_collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
sampler=DistributedSampler(
trainset,
num_replicas=args.world_size,
rank=rank))
val_loader = DataLoader(dataset=valset,
collate_fn=custom_collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
sampler=DistributedSampler(
valset,
num_replicas=args.world_size,
rank=rank))
# Fix random seed for model synchronisation
torch.manual_seed(args.random_seed)
if args.dataset == 'hicodet':
object_to_target = train_loader.dataset.dataset.object_to_verb
human_idx = 49
num_classes = 117
elif args.dataset == 'vcoco':
object_to_target = train_loader.dataset.dataset.object_to_action
human_idx = 1
num_classes = 24
net = SCG(object_to_target,
human_idx,
num_classes=num_classes,
num_iterations=args.num_iter,
postprocess=False,
max_human=args.max_human,
max_object=args.max_object,
box_score_thresh=args.box_score_thresh,
distributed=True)
if os.path.exists(args.checkpoint_path):
print("=> Rank {}: continue from saved checkpoint".format(rank),
args.checkpoint_path)
checkpoint = torch.load(args.checkpoint_path, map_location='cpu')
net.load_state_dict(checkpoint['model_state_dict'])
optim_state_dict = checkpoint['optim_state_dict']
sched_state_dict = checkpoint['scheduler_state_dict']
epoch = checkpoint['epoch']
iteration = checkpoint['iteration']
else:
print(
"=> Rank {}: start from a randomly initialised model".format(rank))
optim_state_dict = None
sched_state_dict = None
epoch = 0
iteration = 0
engine = CustomisedDLE(net,
train_loader,
val_loader,
num_classes=num_classes,
print_interval=args.print_interval,
cache_dir=args.cache_dir)
# Seperate backbone parameters from the rest
param_group_1 = []
param_group_2 = []
for k, v in engine.fetch_state_key('net').named_parameters():
if v.requires_grad:
if k.startswith('module.backbone'):
param_group_1.append(v)
elif k.startswith('module.interaction_head'):
param_group_2.append(v)
else:
raise KeyError(f"Unknown parameter name {k}")
# Fine-tune backbone with lower learning rate
optim = torch.optim.AdamW([{
'params': param_group_1,
'lr': args.learning_rate * args.lr_decay
}, {
'params': param_group_2
}],
lr=args.learning_rate,
weight_decay=args.weight_decay)
lambda1 = lambda epoch: 1. if epoch < args.milestones[0] else args.lr_decay
lambda2 = lambda epoch: 1. if epoch < args.milestones[0] else args.lr_decay
lr_scheduler = torch.optim.lr_scheduler.LambdaLR(
optim, lr_lambda=[lambda1, lambda2])
# Override optimiser and learning rate scheduler
engine.update_state_key(optimizer=optim, lr_scheduler=lr_scheduler)
engine.update_state_key(epoch=epoch, iteration=iteration)
engine(args.num_epochs)
def main(rank, args):
dist.init_process_group(backend="nccl",
init_method="env://",
world_size=args.world_size,
rank=rank)
trainset = DataFactory(name=args.dataset,
partition=args.partitions[0],
data_root=args.data_root,
detection_root=args.train_detection_dir,
box_score_thresh_h=args.human_thresh,
box_score_thresh_o=args.object_thresh,
flip=True)
valset = DataFactory(name=args.dataset,
partition=args.partitions[1],
data_root=args.data_root,
detection_root=args.val_detection_dir,
box_score_thresh_h=args.human_thresh,
box_score_thresh_o=args.object_thresh)
train_loader = DataLoader(dataset=trainset,
collate_fn=custom_collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
sampler=DistributedSampler(
trainset,
num_replicas=args.world_size,
rank=rank))
val_loader = DataLoader(dataset=valset,
collate_fn=custom_collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=True,
sampler=DistributedSampler(
valset,
num_replicas=args.world_size,
rank=rank))
# Fix random seed for model synchronisation
torch.manual_seed(args.random_seed)
if args.dataset == 'hicodet':
object_to_target = train_loader.dataset.dataset.object_to_verb
human_idx = 49
num_classes = 117
elif args.dataset == 'vcoco':
object_to_target = train_loader.dataset.dataset.object_to_action
human_idx = 1
num_classes = 24
net = SpatioAttentiveGraph(object_to_target,
human_idx,
num_iterations=args.num_iter,
postprocess=False,
max_human=args.max_human,
max_object=args.max_object)
# Fix backbone parameters
for p in net.backbone.parameters():
p.requires_grad = False
if os.path.exists(args.checkpoint_path):
print("=> Rank {}: continue from saved checkpoint".format(rank),
args.checkpoint_path)
checkpoint = torch.load(args.checkpoint_path, map_location='cpu')
net.load_state_dict(checkpoint['model_state_dict'])
optim_state_dict = checkpoint['optim_state_dict']
sched_state_dict = checkpoint['scheduler_state_dict']
epoch = checkpoint['epoch']
iteration = checkpoint['iteration']
else:
print(
"=> Rank {}: start from a randomly initialised model".format(rank))
optim_state_dict = None
sched_state_dict = None
epoch = 0
iteration = 0
engine = CustomisedDLE(net,
train_loader,
val_loader,
num_classes=num_classes,
optim_params={
'lr': args.learning_rate,
'momentum': args.momentum,
'weight_decay': args.weight_decay
},
optim_state_dict=optim_state_dict,
lr_scheduler=True,
lr_sched_params={
'milestones': args.milestones,
'gamma': args.lr_decay
},
print_interval=args.print_interval,
cache_dir=args.cache_dir)
engine.update_state_key(epoch=epoch, iteration=iteration)
if sched_state_dict is not None:
engine.fetch_state_key('lr_scheduler').load_state_dict(
sched_state_dict)
engine(args.num_epochs)
def __init__(self, config, graph, if_training=True):
""" Build up the graph
Inputs
------
config :
* batch_size : mini batch size
* log_dir : path to save training summary
* learning_rate : adam's learning rate
* seq_length : length of sequence during training
* penalty_lambda = gradient penalty's weight, ref from paper 'improved-wgan'
* n_resblock : number of resblock in network body
* if_handcraft_features : if_handcraft_features
* residual_alpha : residual block = F(x) * residual_alpha + x
* leaky_relu_alpha : tf.maximum(x, leaky_relu_alpha * x)
* openshot_penalty_lambda : Critic = Critic - openshot_penalty_lambda * open_shot_score
* if_use_mismatched : if True, negative scores = mean of (fake_scores + mismatched_scores)
* n_filters : number of filters in all ConV
graph :
tensorflow default graph
"""
self.data_factory = DataFactory()
# hyper-parameters
self.batch_size = config.batch_size
self.log_dir = config.log_dir
self.learning_rate = config.learning_rate
self.seq_length = config.seq_length
self.penalty_lambda = config.penalty_lambda
self.n_resblock = config.n_resblock
self.if_handcraft_features = config.if_handcraft_features
self.residual_alpha = config.residual_alpha
self.leaky_relu_alpha = config.leaky_relu_alpha
self.openshot_penalty_lambda = config.openshot_penalty_lambda
self.if_use_mismatched = config.if_use_mismatched
self.n_filters = config.n_filters
self.if_training = if_training
# steps
self.__global_steps = tf.train.get_or_create_global_step(graph=graph)
with tf.name_scope('Critic'):
self.__steps = tf.get_variable(
'C_steps',
shape=[],
dtype=tf.int32,
initializer=tf.zeros_initializer(dtype=tf.int32),
trainable=False)
# data
self.__G_samples = tf.placeholder(dtype=tf.float32,
shape=[None, None, 10],
name='G_samples')
self.__real_data = tf.placeholder(dtype=tf.float32,
shape=[None, None, 10],
name='real_data')
self.__matched_cond = tf.placeholder(dtype=tf.float32,
shape=[None, None, 13],
name='matched_cond')
self.__mismatched_cond = tf.random_shuffle(self.__matched_cond)
# adversarial learning : wgan
self.__build_model()
# summary
if self.if_training:
self.__summary_op = tf.summary.merge(tf.get_collection('C'))
self.__summary_histogram_op = tf.summary.merge(
tf.get_collection('C_histogram'))
self.__summary_valid_op = tf.summary.merge(
tf.get_collection('C_valid'))
self.summary_writer = tf.summary.FileWriter(self.log_dir + 'C')
self.valid_summary_writer = tf.summary.FileWriter(
self.log_dir + 'C_valid')
else:
self.baseline_summary_writer = tf.summary.FileWriter(
self.log_dir + 'Baseline_C')
def __init__(self, config, graph, if_training=True):
""" TO build up the graph
Inputs
------
config :
* batch_size : mini batch size
* log_dir : path to save training summary
* learning_rate : adam's learning rate
* seq_length : length of LSTM
* latent_dims : dimensions of latent feature
* penalty_lambda = gradient penalty's weight, ref from paper of 'improved-wgan'
graph :
tensorflow default graph
"""
self.data_factory = DataFactory()
# hyper-parameters
self.batch_size = config.batch_size
self.log_dir = config.log_dir
self.learning_rate = config.learning_rate
self.seq_length = config.seq_length
self.latent_dims = config.latent_dims
# RNN parameters
self.num_features = config.num_features
self.num_layers = config.num_layers
self.hidden_size = config.hidden_size
self.if_training = if_training
self.penalty_lambda = config.penalty_lambda
self.if_handcraft_features = config.if_handcraft_features
self.leaky_relu_alpha = config.leaky_relu_alpha
self.heuristic_penalty_lambda = config.heuristic_penalty_lambda
self.if_use_mismatched = config.if_use_mismatched
self.if_trainable_lambda = config.if_trainable_lambda
# steps
self.__global_steps = tf.train.get_or_create_global_step(graph=graph)
self.__steps = 0
# data
self.__G_samples = tf.placeholder(
dtype=tf.float32,
shape=[self.batch_size, self.seq_length, 10],
name='G_samples')
self.__real_data = tf.placeholder(
dtype=tf.float32,
shape=[self.batch_size, self.seq_length, 10],
name='real_data')
self.__matched_cond = tf.placeholder(
dtype=tf.float32,
shape=[self.batch_size, self.seq_length, 13],
name='matched_cond')
self.__mismatched_cond = tf.random_shuffle(self.__matched_cond)
# adversarial learning : wgan
self.__build_model()
# summary
if self.if_training:
self.__summary_op = tf.summary.merge(tf.get_collection('C'))
# self.__summary_histogram_op = tf.summary.merge(
# tf.get_collection('C_histogram'))
self.__summary_valid_op = tf.summary.merge(
tf.get_collection('C_valid'))
self.summary_writer = tf.summary.FileWriter(self.log_dir + 'C')
self.valid_summary_writer = tf.summary.FileWriter(self.log_dir +
'C_valid')
else:
self.baseline_summary_writer = tf.summary.FileWriter(self.log_dir +
'Baseline_C')
def mode_4(sess, graph, save_path, is_valid=FLAGS.is_valid):
""" to analize code, only change first dimension for comparison
Saved Result
------------
results_A_fake_B : float, numpy ndarray, shape=[n_latents=11, n_conditions=128*9, length=100, features=23]
Real A + Fake B
results_A_real_B : float, numpy ndarray, shape=[n_latents=11, n_conditions=128*9, length=100, features=23]
Real A + Real B
results_critic_scores : float, numpy ndarray, shape=[n_latents=11, n_conditions=128*9]
critic scores for each input data
"""
target_dims = 0
n_latents = 11
# placeholder tensor
latent_input_t = graph.get_tensor_by_name('latent_input:0')
team_a_t = graph.get_tensor_by_name('team_a:0')
G_samples_t = graph.get_tensor_by_name('G_samples:0')
matched_cond_t = graph.get_tensor_by_name('matched_cond:0')
# result tensor
result_t = graph.get_tensor_by_name(
'Generator/G_inference/conv_result/conv1d/Maximum:0')
critic_scores_t = graph.get_tensor_by_name(
'Critic/C_inference_1/linear_result/BiasAdd:0')
# 'Generator/G_loss/C_inference/linear_result/Reshape:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
# normalize
data_factory = DataFactory(real_data)
# result collector
results_A_fake_B = []
results_A_real_B = []
results_critic_scores = []
# shuffle the data
train_data, valid_data = data_factory.fetch_data()
if is_valid:
target_data = valid_data
else:
target_data = train_data
latents = z_samples(FLAGS.batch_size)
for idx in range(0, FLAGS.n_conditions, FLAGS.batch_size):
real_samples = target_data['B'][idx:idx + FLAGS.batch_size]
real_conds = target_data['A'][idx:idx + FLAGS.batch_size]
# generate result
temp_critic_scores = []
temp_A_fake_B = []
for i in range(n_latents):
latents[:, target_dims] = -2.5 + 0.5 * i
feed_dict = {
latent_input_t: latents,
team_a_t: real_conds
}
result = sess.run(
result_t, feed_dict=feed_dict)
feed_dict = {
G_samples_t: result,
matched_cond_t: real_conds
}
critic_scores = sess.run(
critic_scores_t, feed_dict=feed_dict)
temp_A_fake_B.append(data_factory.recover_data(
np.concatenate([real_conds, result], axis=-1)))
temp_critic_scores.append(critic_scores)
results_A_fake_B.append(temp_A_fake_B)
results_critic_scores.append(temp_critic_scores)
# concat along with conditions dimension (axis=1)
results_A_fake_B = np.concatenate(results_A_fake_B, axis=1)
results_critic_scores = np.concatenate(results_critic_scores, axis=1)
results_A = data_factory.recover_BALL_and_A(
target_data['A'][:FLAGS.n_conditions])
results_real_B = data_factory.recover_B(
target_data['B'][:FLAGS.n_conditions])
results_A_real_B = np.concatenate([results_A, results_real_B], axis=-1)
# saved as numpy
print(np.array(results_A_fake_B).shape)
print(np.array(results_A_real_B).shape)
print(np.array(results_critic_scores).shape)
np.save(save_path + 'results_A_fake_B.npy',
np.array(results_A_fake_B).astype(np.float32).reshape([n_latents, FLAGS.n_conditions, FLAGS.seq_length, 23]))
np.save(save_path + 'results_A_real_B.npy',
np.array(results_A_real_B).astype(np.float32).reshape([FLAGS.n_conditions, FLAGS.seq_length, 23]))
np.save(save_path + 'results_critic_scores.npy',
np.array(results_critic_scores).astype(np.float32).reshape([n_latents, FLAGS.n_conditions]))
print('!!Completely Saved!!')
def mode_9(sess, graph, save_path, is_valid=FLAGS.is_valid):
""" to collect results vary in length
Saved Result
------------
results_A_fake_B : float, numpy ndarray, shape=[n_latents=100, n_conditions=100, length=100, features=23]
Real A + Fake B
results_A_real_B : float, numpy ndarray, shape=[n_conditions=100, length=100, features=23]
Real A + Real B
results_critic_scores : float, numpy ndarray, shape=[n_latents=100, n_conditions=100]
critic scores for each input data
"""
# placeholder tensor
latent_input_t = graph.get_tensor_by_name('Generator/latent_input:0')
team_a_t = graph.get_tensor_by_name('Generator/team_a:0')
G_samples_t = graph.get_tensor_by_name('Critic/G_samples:0')
matched_cond_t = graph.get_tensor_by_name('Critic/matched_cond:0')
# result tensor
result_t = graph.get_tensor_by_name(
'Generator/G_inference/conv_result/conv1d/Maximum:0')
critic_scores_t = graph.get_tensor_by_name(
'Critic/C_inference_1/conv_output/Reshape:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
# DataFactory
data_factory = DataFactory(real_data)
# target data
target_data = np.load('../../data/FixedFPS5.npy')[-100:]
target_length = np.load('../../data/FixedFPS5Length.npy')[-100:]
print('target_data.shape', target_data.shape)
team_AB = np.concatenate(
[
# ball
target_data[:, :, 0, :3].reshape(
[target_data.shape[0], target_data.shape[1], 1 * 3]),
# team A players
target_data[:, :, 1:6, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2]),
# team B players
target_data[:, :, 6:11, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2])
],
axis=-1)
team_AB = data_factory.normalize(team_AB)
team_A = team_AB[:, :, :13]
team_B = team_AB[:, :, 13:]
# result collector
results_A_fake_B = []
results_A_real_B = []
results_critic_scores = []
for idx in range(team_AB.shape[0]):
# given 100(FLAGS.n_latents) latents generate 100 results on same condition at once
real_samples = team_B[idx:idx + 1, :target_length[idx]]
real_samples = np.concatenate(
[real_samples for _ in range(FLAGS.n_latents)], axis=0)
real_conds = team_A[idx:idx + 1, :target_length[idx]]
real_conds = np.concatenate(
[real_conds for _ in range(FLAGS.n_latents)], axis=0)
# generate result
latents = z_samples(FLAGS.n_latents)
feed_dict = {latent_input_t: latents, team_a_t: real_conds}
result = sess.run(result_t, feed_dict=feed_dict)
# calculate em distance
feed_dict = {G_samples_t: result, matched_cond_t: real_conds}
em_dist = sess.run(critic_scores_t, feed_dict=feed_dict)
recoverd_A_fake_B = data_factory.recover_data(
np.concatenate([real_conds, result], axis=-1))
# padding to length=200
dummy = np.zeros(shape=[
FLAGS.n_latents, team_AB.shape[1] -
target_length[idx], team_AB.shape[2]
])
temp_A_fake_B_concat = np.concatenate([recoverd_A_fake_B, dummy],
axis=1)
results_A_fake_B.append(temp_A_fake_B_concat)
results_critic_scores.append(em_dist)
print(np.array(results_A_fake_B).shape)
print(np.array(results_critic_scores).shape)
# concat along with conditions dimension (axis=1)
results_A_fake_B = np.stack(results_A_fake_B, axis=1)
results_critic_scores = np.stack(results_critic_scores, axis=1)
# real data
results_A = data_factory.recover_BALL_and_A(team_A)
results_real_B = data_factory.recover_B(team_B)
results_A_real_B = data_factory.recover_data(team_AB)
# saved as numpy
print(np.array(results_A_fake_B).shape)
print(np.array(results_A_real_B).shape)
print(np.array(results_critic_scores).shape)
np.save(
save_path + 'results_A_fake_B.npy',
np.array(results_A_fake_B).astype(np.float32).reshape(
[FLAGS.n_latents, team_AB.shape[0], team_AB.shape[1], 23]))
np.save(
save_path + 'results_A_real_B.npy',
np.array(results_A_real_B).astype(np.float32).reshape(
[team_AB.shape[0], team_AB.shape[1], 23]))
np.save(
save_path + 'results_critic_scores.npy',
np.array(results_critic_scores).astype(np.float32).reshape(
[FLAGS.n_latents, team_AB.shape[0]]))
print('!!Completely Saved!!')
def mode_1(sess, graph, save_path, is_valid=FLAGS.is_valid):
""" to collect results
Saved Result
------------
results_A_fake_B : float, numpy ndarray, shape=[n_latents=100, n_conditions=128*9, length=100, features=23]
Real A + Fake B
results_A_real_B : float, numpy ndarray, shape=[n_conditions=128*9, length=100, features=23]
Real A + Real B
results_critic_scores : float, numpy ndarray, shape=[n_latents=100, n_conditions=128*9]
critic scores for each input data
"""
# placeholder tensor
latent_input_t = graph.get_tensor_by_name('latent_input:0')
team_a_t = graph.get_tensor_by_name('team_a:0')
G_samples_t = graph.get_tensor_by_name('G_samples:0')
matched_cond_t = graph.get_tensor_by_name('matched_cond:0')
# result tensor
result_t = graph.get_tensor_by_name(
'Generator/G_inference/conv_result/conv1d/Maximum:0')
# critic_scores_t = graph.get_tensor_by_name(
# 'Critic/C_inference_1/conv_output/Reshape:0')
critic_scores_t = graph.get_tensor_by_name(
'Critic/C_inference_1/linear_result/BiasAdd:0')
if not os.path.exists(save_path):
os.makedirs(save_path)
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
# normalize
data_factory = DataFactory(real_data)
# result collector
results_A_fake_B = []
results_A_real_B = []
results_critic_scores = []
# shuffle the data
train_data, valid_data = data_factory.fetch_data()
if is_valid:
target_data = valid_data
else:
target_data = train_data
target_data = np.load('ADD-100.npy')
team_AB = np.concatenate(
[
# ball
target_data[:, :, 0, :3].reshape(
[target_data.shape[0], target_data.shape[1], 1 * 3]),
# team A players
target_data[:, :, 1:6, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2]),
# team B players
target_data[:, :, 6:11, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2])
], axis=-1
)
team_AB = data_factory.normalize(team_AB)
print(team_AB.shape)
dummy_AB = np.zeros(shape=[98, 100, 23])
team_AB = np.concatenate([team_AB, dummy_AB], axis=0)
team_A = team_AB[:, :, :13]
team_B = team_AB[:, :, 13:]
# for idx in range(0, FLAGS.batch_size, FLAGS.batch_size):
real_samples = team_B
real_conds = team_A
# generate result
temp_critic_scores = []
temp_A_fake_B = []
for i in range(FLAGS.n_latents):
latents = z_samples(FLAGS.batch_size)
feed_dict = {
latent_input_t: latents,
team_a_t: real_conds
}
result = sess.run(
result_t, feed_dict=feed_dict)
feed_dict = {
G_samples_t: result,
matched_cond_t: real_conds
}
critic_scores = sess.run(
critic_scores_t, feed_dict=feed_dict)
temp_A_fake_B.append(data_factory.recover_data(
np.concatenate([real_conds, result], axis=-1)))
temp_critic_scores.append(critic_scores)
results_A_fake_B.append(temp_A_fake_B)
results_critic_scores.append(temp_critic_scores)
# concat along with conditions dimension (axis=1)
results_A_fake_B = np.concatenate(results_A_fake_B, axis=1)
results_critic_scores = np.concatenate(results_critic_scores, axis=1)
results_A = data_factory.recover_BALL_and_A(
real_conds)
results_real_B = data_factory.recover_B(
real_samples)
results_A_real_B = np.concatenate([results_A, results_real_B], axis=-1)
# saved as numpy
print(np.array(results_A_fake_B).shape)
print(np.array(results_A_real_B).shape)
print(np.array(results_critic_scores).shape)
np.save(save_path + 'results_A_fake_B.npy',
np.array(results_A_fake_B)[:, :30].astype(np.float32).reshape([FLAGS.n_latents, 30, FLAGS.seq_length, 23]))
np.save(save_path + 'results_A_real_B.npy',
np.array(results_A_real_B)[:30].astype(np.float32).reshape([30, FLAGS.seq_length, 23]))
np.save(save_path + 'results_critic_scores.npy',
np.array(results_critic_scores)[:, :30].astype(np.float32).reshape([FLAGS.n_latents, 30]))
print('!!Completely Saved!!')
def rnn():
""" to collect results vary in length
Saved Result
------------
results_A_fake_B : float, numpy ndarray, shape=[n_latents=100, n_conditions=100, length=100, features=23]
Real A + Fake B
results_A_real_B : float, numpy ndarray, shape=[n_conditions=100, length=100, features=23]
Real A + Real B
results_critic_scores : float, numpy ndarray, shape=[n_latents=100, n_conditions=100]
critic scores for each input data
"""
save_path = os.path.join(COLLECT_PATH, 'rnn')
if not os.path.exists(save_path):
os.makedirs(save_path)
real_data = np.load(FLAGS.data_path)[:, :FLAGS.seq_length, :, :]
print('real_data.shape', real_data.shape)
# DataFactory
data_factory = DataFactory(real_data)
# target data
target_data = np.load('../../data/FixedFPS5.npy')[-100:]
target_length = np.load('../../data/FixedFPS5Length.npy')[-100:]
print('target_data.shape', target_data.shape)
team_AB = np.concatenate(
[
# ball
target_data[:, :, 0, :3].reshape(
[target_data.shape[0], target_data.shape[1], 1 * 3]),
# team A players
target_data[:, :, 1:6, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2]),
# team B players
target_data[:, :, 6:11, :2].reshape(
[target_data.shape[0], target_data.shape[1], 5 * 2])
], axis=-1
)
team_AB = data_factory.normalize(team_AB)
team_A = team_AB[:, :, :13]
team_B = team_AB[:, :, 13:]
# result collector
results_A_fake_B = []
results_A_real_B = []
config = TrainingConfig(235)
with tf.get_default_graph().as_default() as graph:
# model
C = C_MODEL(config, graph)
G = G_MODEL(config, C.inference, graph)
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
default_sess = tf.Session(config=tfconfig, graph=graph)
# saver for later restore
saver = tf.train.Saver(max_to_keep=0) # 0 -> keep them all
# restore model if exist
saver.restore(default_sess, FLAGS.restore_path)
print('successfully restore model from checkpoint: %s' %
(FLAGS.restore_path))
for idx in range(team_AB.shape[0]):
# given 100(FLAGS.n_latents) latents generate 100 results on same condition at once
real_samples = team_B[idx:idx + 1, :]
real_samples = np.concatenate(
[real_samples for _ in range(FLAGS.n_latents)], axis=0)
real_conds = team_A[idx:idx + 1, :]
real_conds = np.concatenate(
[real_conds for _ in range(FLAGS.n_latents)], axis=0)
# generate result
latents = z_samples(FLAGS.n_latents)
result = G.generate(default_sess, latents, real_conds)
# calculate em distance
recoverd_A_fake_B = data_factory.recover_data(
np.concatenate([real_conds, result], axis=-1))
# padding to length=200
dummy = np.zeros(
shape=[FLAGS.n_latents, team_AB.shape[1] - target_length[idx], team_AB.shape[2]])
temp_A_fake_B_concat = np.concatenate(
[recoverd_A_fake_B[:, :target_length[idx]], dummy], axis=1)
results_A_fake_B.append(temp_A_fake_B_concat)
print(np.array(results_A_fake_B).shape)
# concat along with conditions dimension (axis=1)
results_A_fake_B = np.stack(results_A_fake_B, axis=1)
# real data
results_A = data_factory.recover_BALL_and_A(team_A)
results_real_B = data_factory.recover_B(team_B)
results_A_real_B = data_factory.recover_data(team_AB)
# saved as numpy
print(np.array(results_A_fake_B).shape)
print(np.array(results_A_real_B).shape)
np.save(os.path.join(save_path, 'results_A_fake_B.npy'),
np.array(results_A_fake_B).astype(np.float32).reshape([FLAGS.n_latents, team_AB.shape[0], team_AB.shape[1], 23]))
np.save(os.path.join(save_path, 'results_A_real_B.npy'),
np.array(results_A_real_B).astype(np.float32).reshape([team_AB.shape[0], team_AB.shape[1], 23]))
print('!!Completely Saved!!')