class ModelTester:
"""
Handles the model interactions
"""
def __init__(self,
image_size,
latent_size,
output_size=7,
num_channels=3,
task='5002',
tasks=['5001', '5002', '5003', '5004'],
save_dir="model/"):
self.image_size = image_size
self.latent_size = latent_size
self.num_channels = num_channels
self.save_dir = save_dir
self.output_size = output_size
self.batch_size = 1
self.task = task
self.tasks = tasks
self.attention_latent_size = 32
self.attention_size = 28
self.auto_regressive = False
self.use_all_cameras = False
self.use_all_cameras_stacked_on_channel = False
if self.use_all_cameras_stacked_on_channel:
self.num_channels = self.num_channels * 3
self.which_camera_to_use = 1
self.dataset_ctrl = TestNetworkRobot(
self.image_size,
config['record_path'],
config['robot_command_file'],
config['camera_topics'],
cache_size=1,
cameras_switch=[False, True, False])
# while not all(self.dataset_ctrl.all_caches_filled):
# print 'Waiting for the caches to fill!'
# time.sleep(1)
DC = DatasetController(batch_size=32,
sequence_input=1,
sequence_output=0,
read_jpgs=True)
self.g = DC.get_next_batch(sth_sth=False,
task=['5002', '5001'],
from_start=False,
train=True,
camera='camera-' +
str(self.which_camera_to_use))
self.num_descs = DC.num_all_objects_describtors
self.num_objects = DC.num_all_objects
self.enc_model = autoencoder.Encoder_text_tower(
density=8,
size=self.image_size,
latent_size=self.latent_size,
channel=self.num_channels,
num_objects=self.num_objects,
num_descriptions=self.num_descs)
self.gen_model = autoencoder.Generator_text(
density=8,
size=image_size,
latent_size=latent_size,
channel=self.num_channels * 2,
num_objects=self.num_objects,
num_descriptions=self.num_descs)
self.mdn_model = RobotController(
(latent_size) + self.num_descs + self.num_objects,
hidden_dimension,
output_size,
num_mixture,
auto_regressive=self.auto_regressive)
self.dataset_path = '/home/d3gan/development/datasets/record/sth_sth_224'
self.source_dataset_path = dataset_path
self.bag_of_words = {}
self.annotations = {}
self.reverse_annotations = {}
self.string_size = 10
self.vocabularySize = 32
self.annotated_tasks = [5001, 5002]
self.bag_of_words = self.fill_bag_of_words(self.annotated_tasks)
for task in self.annotated_tasks:
self.read_annotations(task)
# raw_sentence = "push white plate from left to right"
raw_sentence = "pick-up black dumble"
self.sentence = self.encode_annotation(task, raw_sentence)
for i, label in enumerate(self.sentence):
if int(label) in DC.objects:
self.which_object = DC.objects[int(label)]
if int(label) in DC.objects_describtors:
self.which_describtor = DC.objects_describtors[int(label)]
self.objects_norm = np.zeros((1), dtype=np.float32)
self.objects_norm[0] = (float(self.which_object) /
(self.num_objects + 1)) - 1
self.objects_norm = self.objects_norm[:, np.newaxis, np.newaxis,
np.newaxis]
self.objects_norm = np.tile(self.objects_norm,
(1, 1, self.image_size, self.image_size))
self.descriptions_norm = np.zeros((1), dtype=np.float32)
self.descriptions_norm[0] = (float(self.which_describtor) /
(self.num_descs + 1)) - 1
self.descriptions_norm = self.descriptions_norm[:, np.newaxis,
np.newaxis, np.newaxis]
self.descriptions_norm = np.tile(
self.descriptions_norm, (1, 1, self.image_size, self.image_size))
self.object_involved_one_hot = np.zeros((1, self.num_objects))
self.describtors_involved_one_hot = np.zeros((1, self.num_descs))
self.object_involved_one_hot[0, self.which_object - 1] = 1
self.describtors_involved_one_hot[0, self.which_describtor - 1] = 1
self.object_involved_one_hot = np.asarray(self.object_involved_one_hot,
dtype=np.float32)
self.describtors_involved_one_hot = np.asarray(
self.describtors_involved_one_hot, dtype=np.float32)
self.load_model()
self.to_gpu()
self.real_time_test()
def get_task_one_hot_vector(self, task):
one_hot = np.zeros((self.batch_size, len(self.tasks)),
dtype=np.float32)
for i in range(self.batch_size):
one_hot[i][int(task) - 5001] = 1
return one_hot
def encode_annotation(self, task, sentence):
words = sentence.split()
encoded = np.zeros((self.string_size), dtype=int)
for i, word in enumerate(words):
encoded[i] = int(self.bag_of_words[word])
return encoded
def read_annotations(self, task):
self.reverse_annotations[task] = {}
with open(
os.path.join(self.source_dataset_path,
str(task) + '_task_annotation.csv'),
'rb') as csvfile:
spamreader = csv.reader(csvfile, delimiter=',', quotechar='|')
for row in spamreader:
words = row[0].split()
key = ''
for word in words:
key += str(self.bag_of_words[word]) + ' '
key += '0'
self.annotations[key] = row[1:]
for dem in row[1:]:
self.reverse_annotations[task][dem] = key
def fill_bag_of_words(self, tasks):
unique_words = []
max_len = 0
bag = {}
for task in tasks:
with open(
os.path.join(self.source_dataset_path,
str(task) + '_task_annotation.csv'),
'rb') as csvfile:
spamreader = csv.reader(csvfile, delimiter=',', quotechar='|')
for row in spamreader:
words = row[0].split()
if len(words) > max_len:
max_len = len(words)
for word in words:
if word not in unique_words:
unique_words.append(word)
for i, word in enumerate(unique_words):
bag[word] = i + 1
if max_len + 1 > self.string_size:
print(
"ERROR: provided string size is smaller than the biggest annotation!"
)
return bag
def real_time_test(self):
predicted = np.zeros((self.output_size), dtype=np.float32)
self.mdn_model.reset_state()
while True:
input_online_images, _ = self.dataset_ctrl.get_next_batch(
batch_size=1,
camera_id=self.which_camera_to_use,
channel_first=True,
homography=True)
# resized_image = Image.fromarray(np.uint8((input_online_images[0] + 1) * 127.5).transpose((1, 2, 0))).resize((self.image_size, self.image_size), resample=Image.ANTIALIAS)
# # toShow.show()
# # raw_input()
# resized_image = resized_image.convert('RGB')
# resized_image = np.asarray(resized_image, dtype=np.float32)
# resized_image = resized_image.transpose((2, 0, 1))
# resized_image = resized_image / 127.5 - 1
# resized_image = cv2.resize(((input_online_images[0] + 1) * 127.5).transpose(1, 2, 0), dsize=(self.image_size, self.image_size), interpolation=cv2.INTER_CUBIC)
# resized_image = ((resized_image / 127.5) -1).transpose(2, 0, 1)
# self.show_image(resized_image[np.newaxis, :])
# att_latent = self.pca_model.transform(att.reshape(1, -1) - self.pca_data_mean)
# inv_att = self.pca_model.inverse_transform(att_latent) + self.pca_data_mean
# inv_att_0 = np.reshape(inv_att, (28, 28))
# plt.imshow(inv_att_0, cmap='gray')
# plt.show()
# att = np.asarray(att[np.newaxis, np.newaxis, :], dtype=np.float32)
# att_latent = np.asarray(att_latent[np.newaxis, :], dtype=np.float32)
# att_latent = np.swapaxes(att_latent, 0, 1)
# att_0 = np.reshape(att, (28, 28))
# plt.imshow(att_0, cmap='gray')
# plt.show()
# self.show_image(input_online_images)
# input_online_images = input_online_images.transpose(0, 3, 2, 1)
# input_images, _, _, _, _, _, _, _, _, _, _, _, _, _ = next(self.g)
# input_images = np.asarray(input_images, dtype=np.float32)
# input_images = np.squeeze(input_images, axis=1)
# self.show_image(input_images[np.newaxis])
# toshow = resized_image.transpose(1, 2, 0)
# toshow = Image.fromarray(np.uint8((toshow + 1) * 127.5))
# toshow.show()
input_online_images = np.asarray(input_online_images[0],
dtype=np.float32)
# input_images[0] = resized_image
x_in = cuda.to_gpu(input_online_images[np.newaxis], GPU.main_gpu)
z0, mean, var = self.enc_model(
Variable(x_in),
Variable(
cuda.to_gpu(self.object_involved_one_hot, GPU.main_gpu)),
Variable(
cuda.to_gpu(self.describtors_involved_one_hot,
GPU.main_gpu)),
train=False)
input_online_images = np.transpose(input_online_images, (1, 2, 0))
tshow = cv2.cvtColor((input_online_images + 1) * 127.5,
cv2.COLOR_BGR2RGB)
cv2.imshow(
'Input image original',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_NEAREST))
cv2.waitKey(10)
# x_in_train = cuda.to_gpu(input_images, GPU.main_gpu)
# z0_train, _, _ = self.enc_model(Variable(x_in_train), train=False)
x_in_att = self.gen_model(
z0,
Variable(
cuda.to_gpu(self.object_involved_one_hot, GPU.main_gpu)),
Variable(
cuda.to_gpu(self.describtors_involved_one_hot,
GPU.main_gpu)),
train=False)
tshow = (cuda.to_cpu(x_in_att[:, :3].data)[0] + 1) * 127.5
tshow = cv2.cvtColor(tshow.transpose((1, 2, 0)), cv2.COLOR_BGR2RGB)
cv2.imshow(
'Reconstruction att image',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_CUBIC))
cv2.waitKey(1)
tshow = (cuda.to_cpu(x_in_att[:, 3:].data)[0] + 1) * 127.5
tshow = cv2.cvtColor(tshow.transpose((1, 2, 0)), cv2.COLOR_BGR2RGB)
cv2.imshow(
'Reconstruction whole image',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_CUBIC))
cv2.waitKey(1)
# self.show_image(cuda.to_cpu(x_in.data))
task_one_hot = self.get_task_one_hot_vector(self.task)
task_one_hot = Variable(cuda.to_gpu(task_one_hot, GPU.main_gpu))
input_feature = F.concat(
(z0,
Variable(
cuda.to_gpu(self.object_involved_one_hot, GPU.main_gpu)),
Variable(
cuda.to_gpu(self.describtors_involved_one_hot,
GPU.main_gpu))),
axis=-1)
input_feature = F.expand_dims(input_feature, axis=0)
dummy_joints = Variable(
cuda.to_gpu(
np.zeros((1, 1, self.output_size), dtype=np.float32),
GPU.main_gpu))
if self.auto_regressive:
_, sample = self.mdn_model(data_in=task_one_hot,
z=input_feature,
data_out=dummy_joints,
return_sample=True,
train=False)
predicted = sample
else:
_, sample = self.mdn_model(data_in=task_one_hot,
z=input_feature,
data_out=dummy_joints,
return_sample=True,
train=False)
# sample = self.mdn_model(data_in=task_one_hot, z=input_feature, train=False)
predicted = cuda.to_cpu(sample.data)[0]
# self.write_command(sample.data[0])
self.dataset_ctrl.send_command(predicted)
print predicted
# raw_input()
# print input_online_images.shape
def morph_attention_and_image(self, images, attentions):
toRet_att = np.empty((1, self.image_size, self.image_size))
att = np.asarray(attentions, dtype=np.float32)
mins = np.min(att, axis=1)
maxs = np.max(att, axis=1)
att -= mins[:, np.newaxis]
att = (2 * att) / maxs[:, np.newaxis] - 1
att = np.reshape(att, (-1, 28, 28))
att = scipy.ndimage.zoom(att, (1, 28 / 28.0, 28 / 28.0), order=3)
att = att[0]
att = F.max_pooling_2d(Variable(att[np.newaxis, np.newaxis, :]),
6).data
att[att < 0.95] = 0
# plt.imshow(att[0][0], cmap='gray')
# plt.show()
# image = images.transpose((1, 2, 0))
# image = np.transpose(images, (1,2,0))
att = scipy.ndimage.zoom(att[0, 0],
(self.image_size / float(att.shape[2]),
self.image_size / float(att.shape[2])),
order=2)
# toshow = scipy.ndimage.zoom(att, (1, 1), order=1)
# cv2.imshow('att', toshow)
# cv2.waitKey(1)
# att = F.unpooling_2d(Variable(att), 4, cover_all=True).data
# cv2.imshow('image', att*255)
# cv2.waitKey(1000)
images = images * att[:, :, np.newaxis]
# images = image.transpose((2, 0, 1))
# images = np.transpose(image, (2, 0, 1))
toRet_att = att[np.newaxis, :, :]
# im = Image.fromarray(np.uint8(((image + 1) * 127.5)))
# im.show()
return images, toRet_att
def show_image(self, images):
for i in range(images.shape[0]):
# moving axis to use plt: i.e [4,100,100] to [100,100,4]
img = images[i]
img = img.transpose(1, 2, 0)
img = (img + 1) * 127.5
img = img.astype(np.uint8)
print img.dtype, np.max(img), np.min(img), np.shape(img)
img = Image.fromarray(img, "RGB")
img.show()
# raw_input()
def write_command(self, command):
with open(os.path.join(record_path, 'commands.csv'), 'wb') as csvfile:
spamwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(command)
def load_model(self):
try:
file_path = os.path.join(os.path.dirname(__file__), self.save_dir)
serializers.load_hdf5(file_path + 'enc.model', self.enc_model)
# serializers.load_hdf5(file_path + 'enc_adverserial.state', self.optimizer_enc)
# serializers.load_hdf5(file_path + 'dis.model', self.dis_model)
# serializers.load_hdf5(file_path + 'dis.state', self.optimizer_dis)
#
serializers.load_hdf5(file_path + 'gen.model', self.gen_model)
# serializers.load_hdf5(file_path + 'gen.state', self.optimizer_gen)
#
# # serializers.load_hdf5se_path + 'gen_mask.state', self.optimizer_gen)
#
# if self.ae_mode == 'AAE':
# serializers.load_hdf5(file_path + 'aae.model', self.aae_dis_model)
# serializers.load_hdf5(file_path + 'aae.state', self.optimizer_aae_dis)
serializers.load_hdf5(file_path + 'rnn_mdn_adverserial.model',
self.mdn_model)
# serializers.load_hdf5(file_path + 'rnn_mdn.state', self.optimizer_mdn)
print('Models has been loaded!')
except Exception as inst:
print inst
print 'cannot load model from {}'.format(file_path)
sys.exit(0)
def to_gpu(self):
self.enc_model.to_gpu(GPU.main_gpu)
self.gen_model.to_gpu(GPU.main_gpu)
self.mdn_model.to_gpu(GPU.main_gpu)
def __init__(self,
image_size,
latent_size,
output_size=7,
num_channels=3,
task='5002',
tasks=['5001', '5002', '5003', '5004'],
save_dir="model/"):
self.image_size = image_size
self.latent_size = latent_size
self.num_channels = num_channels
self.save_dir = save_dir
self.output_size = output_size
self.batch_size = 1
self.task = task
self.tasks = tasks
self.attention_latent_size = 32
self.attention_size = 28
self.auto_regressive = False
self.use_all_cameras = False
self.use_all_cameras_stacked_on_channel = False
if self.use_all_cameras_stacked_on_channel:
self.num_channels = self.num_channels * 3
self.which_camera_to_use = 1
self.dataset_ctrl = TestNetworkRobot(
self.image_size,
config['record_path'],
config['robot_command_file'],
config['camera_topics'],
cache_size=1,
cameras_switch=[False, True, False])
# while not all(self.dataset_ctrl.all_caches_filled):
# print 'Waiting for the caches to fill!'
# time.sleep(1)
DC = DatasetController(batch_size=32,
sequence_input=1,
sequence_output=0,
read_jpgs=True)
self.g = DC.get_next_batch(sth_sth=False,
task=['5002', '5001'],
from_start=False,
train=True,
camera='camera-' +
str(self.which_camera_to_use))
self.num_descs = DC.num_all_objects_describtors
self.num_objects = DC.num_all_objects
self.enc_model = autoencoder.Encoder_text_tower(
density=8,
size=self.image_size,
latent_size=self.latent_size,
channel=self.num_channels,
num_objects=self.num_objects,
num_descriptions=self.num_descs)
self.gen_model = autoencoder.Generator_text(
density=8,
size=image_size,
latent_size=latent_size,
channel=self.num_channels * 2,
num_objects=self.num_objects,
num_descriptions=self.num_descs)
self.mdn_model = RobotController(
(latent_size) + self.num_descs + self.num_objects,
hidden_dimension,
output_size,
num_mixture,
auto_regressive=self.auto_regressive)
self.dataset_path = '/home/d3gan/development/datasets/record/sth_sth_224'
self.source_dataset_path = dataset_path
self.bag_of_words = {}
self.annotations = {}
self.reverse_annotations = {}
self.string_size = 10
self.vocabularySize = 32
self.annotated_tasks = [5001, 5002]
self.bag_of_words = self.fill_bag_of_words(self.annotated_tasks)
for task in self.annotated_tasks:
self.read_annotations(task)
# raw_sentence = "push white plate from left to right"
raw_sentence = "pick-up black dumble"
self.sentence = self.encode_annotation(task, raw_sentence)
for i, label in enumerate(self.sentence):
if int(label) in DC.objects:
self.which_object = DC.objects[int(label)]
if int(label) in DC.objects_describtors:
self.which_describtor = DC.objects_describtors[int(label)]
self.objects_norm = np.zeros((1), dtype=np.float32)
self.objects_norm[0] = (float(self.which_object) /
(self.num_objects + 1)) - 1
self.objects_norm = self.objects_norm[:, np.newaxis, np.newaxis,
np.newaxis]
self.objects_norm = np.tile(self.objects_norm,
(1, 1, self.image_size, self.image_size))
self.descriptions_norm = np.zeros((1), dtype=np.float32)
self.descriptions_norm[0] = (float(self.which_describtor) /
(self.num_descs + 1)) - 1
self.descriptions_norm = self.descriptions_norm[:, np.newaxis,
np.newaxis, np.newaxis]
self.descriptions_norm = np.tile(
self.descriptions_norm, (1, 1, self.image_size, self.image_size))
self.object_involved_one_hot = np.zeros((1, self.num_objects))
self.describtors_involved_one_hot = np.zeros((1, self.num_descs))
self.object_involved_one_hot[0, self.which_object - 1] = 1
self.describtors_involved_one_hot[0, self.which_describtor - 1] = 1
self.object_involved_one_hot = np.asarray(self.object_involved_one_hot,
dtype=np.float32)
self.describtors_involved_one_hot = np.asarray(
self.describtors_involved_one_hot, dtype=np.float32)
self.load_model()
self.to_gpu()
self.real_time_test()
def __init__(self,
image_size,
latent_size,
batch_size,
sequence_size,
num_channels=3,
save_dir="model/",
epoch_num=320,
sample_image_cols=5,
sample_image_rows=4,
load_models=True):
self.evaluator = evaluation()
self.dataset_ctrl = DatasetController(batch_size=int(batch_size),
sequence_size=sequence_size,
shuffle_att=False)
self.num_tasks = len(config['tasks'])
self.image_size = 128
self.att_size = 16
self.normer = image_size * image_size * 3 * 60
self.vocabularySize = 34
self.text_encoding_size = 32
self.batch_size = batch_size
self.sequence_size = sequence_size
self.num_channels = num_channels
self.latent_size = latent_size
self.hidden_dimension = hidden_dimension
self.num_mixture = num_mixture
self.output_size = output_size
self.save_dir = save_dir
self.epoch_num = epoch_num
self.load_models = load_models
self.last_best_result = 100
self.save_model_period = 500
self.save_evaluation_metrics = 500
self.save_sample_image_interval = 250
self.sample_image_cols = sample_image_cols
self.sample_image_rows = sample_image_rows
self.generator_test = self.dataset_ctrl.get_next_batch(
task=['5002', '5001'],
from_start_prob=0,
train=False,
camera='camera-1',
return_mix=True)
self.num_descs = self.dataset_ctrl.num_all_objects_describtors
self.num_objects = self.dataset_ctrl.num_all_objects
images, images_mix, \
images_single, images_single_mix, \
images_multi, \
_, _, sentence, \
objects, objs_one_hot, descriptions, descs_one_hot = next(self.generator_test)
images = images_single_mix[:, 0]
images_multi = images_multi[:, 0]
images = np.reshape(
images, (-1, self.num_channels, self.image_size, self.image_size))
images_multi = np.reshape(
images_multi,
(-1, self.num_channels, self.image_size, self.image_size))
sample_size = int(self.sample_image_cols * self.sample_image_rows / 2)
temp = np.asarray(images[:sample_size], dtype=np.float32)
self.sample_images = np.concatenate(
(temp, np.asarray(images_multi[:sample_size], dtype=np.float32)),
axis=0)
temp = np.asarray(sentence[:sample_size], dtype=np.float32)
self.sample_sentence = np.concatenate((temp, temp), axis=0)
temp = np.asarray(objs_one_hot[:sample_size], dtype=np.float32)
self.sample_objs_one_hot = np.concatenate((temp, temp), axis=0)
temp = np.asarray(descs_one_hot[:sample_size], dtype=np.float32)
self.sample_descs_one_hot = np.concatenate((temp, temp), axis=0)
print(np.squeeze(objects)[:sample_size])
print(np.squeeze(descriptions)[:sample_size])
self.generator_test = self.dataset_ctrl.get_next_batch(
task=['5002', '5001'],
from_start_prob=0,
train=False,
camera='camera-1',
return_multi=False,
return_mix=True)
self.generator = self.dataset_ctrl.get_next_batch(
task=['5002', '5001'],
from_start_prob=0,
camera='camera-1',
return_single=True,
return_multi=True,
return_mix=True,
return_dem=True)
self.enc_model = autoencoders.reduction_att.Encoder_text_tower(
density=8,
size=self.image_size,
latent_size=latent_size,
att_size=self.att_size,
num_objects=self.num_objects,
num_descriptions=self.num_descs)
self.att_enc_model = autoencoders.tower.Encoder(
density=8,
size=self.image_size,
latent_size=latent_size,
channel=self.num_channels)
self.att_gen_model = autoencoders.reduction_att.Generator(
density=8,
size=image_size,
latent_size=latent_size,
att_size=self.att_size,
channel=2 * self.num_channels,
num_objects=self.num_objects,
num_descriptions=self.num_descs)
self.dis_model = autoencoders.reduction_att.Discriminator(
density=8,
size=self.image_size,
channel=self.num_channels,
num_obj=self.num_objects + 1,
num_desc=self.num_descs + 1)
self.mdn_model = RobotController(self.latent_size + self.num_objects +
self.num_descs + 4,
hidden_dimension,
output_size,
num_mixture,
auto_regressive=False)
self.enc_models = [self.enc_model]
self.att_enc_models = [self.att_enc_model]
self.att_gen_models = [self.att_gen_model]
self.dis_models = [self.dis_model]
self.mdn_models = [self.mdn_model]
self.learning_rate = 0.00001
self.WeightDecay = 0.00001
self.optimizer_enc = optimizers.Adam(alpha=self.learning_rate,
beta1=0.9)
self.optimizer_enc.setup(self.enc_models[0])
self.optimizer_enc.add_hook(
chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_att_enc = optimizers.Adam(alpha=self.learning_rate,
beta1=0.9)
self.optimizer_att_enc.setup(self.att_enc_models[0])
self.optimizer_att_enc.add_hook(
chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_att_gen = optimizers.Adam(alpha=self.learning_rate,
beta1=0.9)
self.optimizer_att_gen.setup(self.att_gen_models[0])
self.optimizer_att_gen.add_hook(
chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_dis = optimizers.Adam(alpha=self.learning_rate,
beta1=0.9)
self.optimizer_dis.setup(self.dis_models[0])
self.optimizer_dis.add_hook(
chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_mdn = optimizers.Adam(alpha=self.learning_rate,
beta1=0.9)
self.optimizer_mdn.setup(self.mdn_models[0])
self.optimizer_mdn.add_hook(chainer.optimizer.WeightDecay(0.00001))
for i in range(GPU.num_gpus - 1):
self.enc_models.append(copy.deepcopy(self.enc_model))
self.att_enc_models.append(copy.deepcopy(self.att_enc_model))
self.att_gen_models.append(copy.deepcopy(self.att_gen_model))
self.dis_models.append(copy.deepcopy(self.dis_model))
self.mdn_models.append(copy.deepcopy(self.mdn_model))
self.batch_gpu_threads = [None] * GPU.num_gpus
if self.load_models:
self.load_model()
self.to_gpu()
def __init__(self):
self.include_sth_sth = False
self.use_all_cameras_stack_on_channel = False
self.use_transformed_images = False
self.source_dataset_path = dataset_path
self.dest_dataset_path = self.source_dataset_path
self.model_path = "model/"
self.image_size = image_size
self.latent_size = latent_size
self.out_filepath = './processed_inputs_' + str(image_size) + '_latent_'+ str(latent_size) + '_top_path_cycled_e2e/'
self.batch_size = 100
self.sequence_size = 15
self.string_size = 10
self.hidden_dim = hidden_dimension
self.cameras = ['camera-0', 'camera-1', 'camera-2']
self.num_mixture = num_mixture
self.dataset_path = dataset_path
self.num_channels = num_channels
self.num_epoch = 10000
self.max_sequence_len = {}
self.save_model_period = 300
self.csv_col_num = csv_col_num
self.output_size = output_size
self.best_test_result = float('Inf')
self.save_dir = "model/"
self.dataset = {}
self.tasks = tasks
self.train_percent = 0.8
self.train_dis = True
self.train_mdn_only = True
self.train_lstm = False
self.tasks_train_on = ['5001', '5002']
self.best_test_loss = 100
self.step_size = 2
self.learning_rate = 0.001
print 'best_test_loss: ' + str(self.best_test_loss)
print 'learning_rate: ' + str(self.learning_rate)
print 'sequence_size: ' + str(self.sequence_size)
print 'step_size: ' + str(self.step_size)
self.output_size = 7
self.annotations = {}
self.reverse_annotations = {}
self.annotated_tasks = [5001, 5002]
self.bag_of_words = self.fill_bag_of_words(self.annotated_tasks)
for task in self.annotated_tasks:
self.read_annotations(task)
self.objects_descriptions = {"white" : 1,
"blue": 2,
"black-white": 3,
"black": 4,
"red": 5}
self.objects = {"plate": 1,
"box": 2,
"qr-box": 3,
"bubble-wrap": 4,
"bowl": 5,
"towel": 6,
"dumble": 7,
"ring": 8}
self.objects_descriptions = {self.bag_of_words[x]:y for x,y in self.objects_descriptions.iteritems()}
self.objects = {self.bag_of_words[x]:y for x,y in self.objects.iteritems()}
self.num_all_objects = len(self.objects.keys())
self.num_all_objects_descriptions = len(self.objects_descriptions.keys())
self.joints_std = {
5001: [0.0, 0.1859, 0.0752, 0.0862, 0.0814, 0.2842, 0.0],
5002: [0.0, 0.2066, 0.0874, 0.0942, 0.0658, 0.2066, 0.0],
5003: [0.0, 0.1723, 0.0741, 0.0936, 0.0651, 0.1722, 0.0],
5004: [0.0, 0.1879, 0.0731, 0.0813, 0.0756, 0.3407, 0.0]
}
# self.mdn_model = MDN_RNN(self.latent_size, self.hidden_dim, self.output_size)
self.mdn_model = RobotController(self.latent_size + self.num_all_objects + self.num_all_objects_descriptions, hidden_dimension, output_size, num_mixture, auto_regressive=False)
# self.dis_model = autoencoder.seq_Discriminator(in_dim=self.latent_size + self.output_size + 4, hidden_dim=64)
self.mdn_models = [self.mdn_model]
# self.dis_models = [self.dis_model]
for _ in range(GPU.num_gpus - 1):
self.mdn_models.append(copy.deepcopy(self.mdn_model))
# self.dis_models.append(copy.deepcopy(self.dis_model))
self.optimizer_mdn = optimizers.Adam(alpha=self.learning_rate, beta1=0.9)
# self.optimizer_mdn = optimizers.RMSpropGraves(lr=self.learning_rate)
self.optimizer_mdn.setup(self.mdn_models[0])
self.optimizer_mdn.add_hook(chainer.optimizer.WeightDecay(0.00001))
# self.optimizer_dis = optimizers.Adam(alpha=self.learning_rate, beta1=0.9)
# self.optimizer_dis.setup(self.dis_models[0])
# self.optimizer_dis.add_hook(chainer.optimizer.WeightDecay(0.00001))
self.batch_gpu_threads = [None] * GPU.num_gpus
self.load_model()
self.to_gpu()
self.load_dataset()
def __init__(self,
image_size,
latent_size,
output_size=7,
num_channels=3,
task='5002',
tasks=['5001', '5002', '5003', '5004'],
save_dir="model/reduction_double_works/"):
self.image_size = image_size
self.latent_size = latent_size
self.num_channels = num_channels
self.save_dir = save_dir
self.output_size = output_size
self.batch_size = 1
self.task = task
self.att_size = 16
self.tasks = tasks
self.attention_latent_size = 32
self.attention_size = 8
self.robot_latent_size = 4
self.text_encoding_size = 32
self.auto_regressive = False
self.use_all_cameras = False
self.use_all_cameras_stacked_on_channel = False
if self.use_all_cameras_stacked_on_channel:
self.num_channels = self.num_channels * 3
self.which_camera_to_use = 1
self.dataset_ctrl = TestNetworkRobot(
self.image_size,
config['record_path'],
config['robot_command_file'],
config['camera_topics'],
cache_size=1,
cameras_switch=[False, True, False])
self.DC = DatasetController(batch_size=32,
sequence_size=1,
read_jpgs=True,
shuffle_att=False)
self.g = self.DC.get_next_batch(task=['5002', '5001'],
from_start_prob=0,
camera='camera-1',
return_multi=True)
self.objects_describtors = {
"white": 1,
"blue": 2,
"black-white": 3,
"black": 4,
"red": 5
}
self.objects = {
"plate": 1,
"box": 2,
"qr-box": 3,
"bubble-wrap": 4,
"bowl": 5,
"towel": 6,
"dumble": 7,
"ring": 8
}
self.num_descs = self.DC.num_all_objects_describtors
self.num_objects = self.DC.num_all_objects
self.enc_model = autoencoders.reduction_att.Encoder_text_tower(
density=8,
size=self.image_size,
latent_size=latent_size,
att_size=self.att_size,
num_objects=self.num_objects,
num_describtions=self.num_descs)
self.att_enc_model = autoencoders.tower.Encoder(
density=8, size=self.image_size, latent_size=latent_size)
self.att_gen_model = autoencoders.reduction_att.Generator(
density=8,
size=image_size,
latent_size=latent_size,
att_size=self.att_size,
channel=2 * self.num_channels,
num_objects=self.num_objects,
num_describtions=self.num_descs)
self.mdn_model = RobotController(self.latent_size + self.num_objects +
self.num_descs + 4,
hidden_dimension,
output_size,
num_mixture_2,
auto_regressive=False)
self.annotated_tasks = [5001, 5002]
# raw_sentence = "push blue box from left to right"
raw_sentence = "pick-up white plate"
self.obj_one_hot = np.zeros(self.num_objects, dtype=np.float32)
self.desc_one_hot = np.zeros(self.num_descs, dtype=np.float32)
for word in raw_sentence.split():
if word in self.objects_describtors:
self.desc_one_hot[self.objects_describtors[word] - 1] = 1
if word in self.objects:
self.obj_one_hot[self.objects[word] - 1] = 1
self.sentence = self.DC.encode_annotation(raw_sentence)
self.sentence = np.asarray(self.DC.sentence_to_one_hot[self.sentence],
dtype=np.float32)
self.load_model()
self.to_gpu()
self.real_time_test()
class ModelTester:
"""
Handles the model interactions
"""
def __init__(self,
image_size,
latent_size,
output_size=7,
num_channels=3,
task='5002',
tasks=['5001', '5002', '5003', '5004'],
save_dir="model/reduction_double_works/"):
self.image_size = image_size
self.latent_size = latent_size
self.num_channels = num_channels
self.save_dir = save_dir
self.output_size = output_size
self.batch_size = 1
self.task = task
self.att_size = 16
self.tasks = tasks
self.attention_latent_size = 32
self.attention_size = 8
self.robot_latent_size = 4
self.text_encoding_size = 32
self.auto_regressive = False
self.use_all_cameras = False
self.use_all_cameras_stacked_on_channel = False
if self.use_all_cameras_stacked_on_channel:
self.num_channels = self.num_channels * 3
self.which_camera_to_use = 1
self.dataset_ctrl = TestNetworkRobot(
self.image_size,
config['record_path'],
config['robot_command_file'],
config['camera_topics'],
cache_size=1,
cameras_switch=[False, True, False])
self.DC = DatasetController(batch_size=32,
sequence_size=1,
read_jpgs=True,
shuffle_att=False)
self.g = self.DC.get_next_batch(task=['5002', '5001'],
from_start_prob=0,
camera='camera-1',
return_multi=True)
self.objects_describtors = {
"white": 1,
"blue": 2,
"black-white": 3,
"black": 4,
"red": 5
}
self.objects = {
"plate": 1,
"box": 2,
"qr-box": 3,
"bubble-wrap": 4,
"bowl": 5,
"towel": 6,
"dumble": 7,
"ring": 8
}
self.num_descs = self.DC.num_all_objects_describtors
self.num_objects = self.DC.num_all_objects
self.enc_model = autoencoders.reduction_att.Encoder_text_tower(
density=8,
size=self.image_size,
latent_size=latent_size,
att_size=self.att_size,
num_objects=self.num_objects,
num_describtions=self.num_descs)
self.att_enc_model = autoencoders.tower.Encoder(
density=8, size=self.image_size, latent_size=latent_size)
self.att_gen_model = autoencoders.reduction_att.Generator(
density=8,
size=image_size,
latent_size=latent_size,
att_size=self.att_size,
channel=2 * self.num_channels,
num_objects=self.num_objects,
num_describtions=self.num_descs)
self.mdn_model = RobotController(self.latent_size + self.num_objects +
self.num_descs + 4,
hidden_dimension,
output_size,
num_mixture_2,
auto_regressive=False)
self.annotated_tasks = [5001, 5002]
# raw_sentence = "push blue box from left to right"
raw_sentence = "pick-up white plate"
self.obj_one_hot = np.zeros(self.num_objects, dtype=np.float32)
self.desc_one_hot = np.zeros(self.num_descs, dtype=np.float32)
for word in raw_sentence.split():
if word in self.objects_describtors:
self.desc_one_hot[self.objects_describtors[word] - 1] = 1
if word in self.objects:
self.obj_one_hot[self.objects[word] - 1] = 1
self.sentence = self.DC.encode_annotation(raw_sentence)
self.sentence = np.asarray(self.DC.sentence_to_one_hot[self.sentence],
dtype=np.float32)
self.load_model()
self.to_gpu()
self.real_time_test()
def get_task_one_hot_vector(self, task):
one_hot = np.zeros((self.batch_size, len(self.tasks)),
dtype=np.float32)
for i in range(self.batch_size):
one_hot[i][int(task) - 5001] = 1
return one_hot
def real_time_test(self):
predicted = np.zeros((self.output_size), dtype=np.float32)
self.mdn_model.reset_state()
seed = None
count = 0
while True:
input_online_images, _ = self.dataset_ctrl.get_next_batch(
batch_size=1,
camera_id=self.which_camera_to_use,
channel_first=True,
homography=True)
obj_one_hot = Variable(
cuda.to_gpu(self.obj_one_hot[np.newaxis], GPU.main_gpu))
desc_one_hot = Variable(
cuda.to_gpu(self.desc_one_hot[np.newaxis], GPU.main_gpu))
input_online_images = np.asarray(input_online_images[0],
dtype=np.float32)
x_in = cuda.to_gpu(input_online_images[np.newaxis], GPU.main_gpu)
att, _, _ = self.enc_model(Variable(x_in),
obj_one_hot,
desc_one_hot,
train=False)
att = F.reshape(att, (-1, 1, self.att_size, self.att_size))
att = F.resize_images(att, (self.image_size, self.image_size))
masked = Variable(x_in) * att
z, mean, var, _ = self.att_enc_model(masked, train=False)
input_online_images = np.transpose(input_online_images, (1, 2, 0))
tshow = cv2.cvtColor((input_online_images + 1) * 127.5,
cv2.COLOR_BGR2RGB)
cv2.imshow(
'Input image original(O)',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_NEAREST))
cv2.moveWindow('Input image original(O)', 0, 0)
cv2.waitKey(10)
tshow = (cuda.to_cpu(masked.data)[0] + 1) * 127.5
tshow = cv2.cvtColor(tshow.transpose((1, 2, 0)), cv2.COLOR_BGR2RGB)
cv2.imshow(
'masked image(M)',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_CUBIC))
cv2.moveWindow('masked image(M)', 450, 0)
cv2.waitKey(1)
x_in_att = self.att_gen_model(z, train=False)
tshow = (cuda.to_cpu(x_in_att[:, :3].data)[0] + 1) * 127.5
tshow = cv2.cvtColor(tshow.transpose((1, 2, 0)), cv2.COLOR_BGR2RGB)
cv2.imshow(
'Reconstruction whole image(O\')',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_CUBIC))
cv2.moveWindow('Reconstruction whole image(O\')', 850, 0)
cv2.waitKey(1)
tshow = (cuda.to_cpu(x_in_att[:, 3:].data)[0] + 1) * 127.5
tshow = cv2.cvtColor(tshow.transpose((1, 2, 0)), cv2.COLOR_BGR2RGB)
cv2.imshow(
'Reconstruction masked image(M\')',
cv2.resize(tshow.astype(np.uint8), (0, 0),
fx=2,
fy=2,
interpolation=cv2.INTER_CUBIC))
cv2.moveWindow('Reconstruction masked image(M\')', 1250, 0)
cv2.waitKey(1)
task_one_hot = self.get_task_one_hot_vector(self.task)
task_one_hot = Variable(cuda.to_gpu(task_one_hot, GPU.main_gpu))
text_encoding = F.concat((task_one_hot, obj_one_hot, desc_one_hot),
axis=-1)
input_feature = z
input_feature = F.expand_dims(input_feature, axis=0)
if count % 60 == 0:
self.mdn_model.reset_state()
print '########################RESET####################################'
dummy_joints = Variable(
cuda.to_gpu(
np.zeros((1, 1, self.output_size), dtype=np.float32),
GPU.main_gpu))
_, sample = self.mdn_model(task_encoding=text_encoding,
image_encoding=input_feature,
data_out=dummy_joints,
return_sample=True,
train=False)
seed = sample
predicted = cuda.to_cpu(sample.data)[0]
count += 1
self.dataset_ctrl.send_command(predicted)
print(predicted)
def show_image(self, images):
for i in range(images.shape[0]):
img = images[i]
img = img.transpose(1, 2, 0)
img = (img + 1) * 127.5
img = img.astype(np.uint8)
print(img.dtype, np.max(img), np.min(img), np.shape(img))
img = Image.fromarray(img, "RGB")
img.show()
def write_command(self, command):
with open(os.path.join(record_path, 'commands.csv'), 'wb') as csvfile:
spamwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(command)
def load_model(self):
try:
file_path = os.path.join(os.path.dirname(__file__), self.save_dir)
serializers.load_hdf5(file_path + 'enc.model', self.enc_model)
serializers.load_hdf5(file_path + 'att_enc.model',
self.att_enc_model)
serializers.load_hdf5(file_path + 'att_gen.model',
self.att_gen_model)
serializers.load_hdf5(file_path + 'rnn_mdn.model', self.mdn_model)
print('Models has been loaded!')
except Exception as inst:
print(inst)
print('cannot load model from {}'.format(file_path))
sys.exit(0)
def to_gpu(self):
self.enc_model.to_gpu(GPU.main_gpu)
self.att_enc_model.to_gpu(GPU.main_gpu)
self.att_gen_model.to_gpu(GPU.main_gpu)
self.mdn_model.to_gpu(GPU.main_gpu)
def __init__(self, image_size, latent_size, batch_size, sequence_size,
num_channels=3, save_dir="model/", epoch_num=320,
sample_image_cols=5, sample_image_rows=2, load_models=True):
self.dataset_ctrl = DatasetController(batch_size=int(batch_size), sequence_input=sequence_size, sequence_output=0, read_jpgs=True)
self.num_tasks = len(config['tasks'])
self.image_size = image_size
self.normer = image_size * image_size * 3 * 60
self.batch_size = batch_size
self.sequence_size = sequence_size
self.num_channels = num_channels
self.latent_size = latent_size
self.hidden_dimension = hidden_dimension
self.num_mixture = num_mixture
self.output_size = output_size
self.save_dir = save_dir
self.epoch_num = epoch_num
self.train_lstm_alone = False
self.train_lstm = False or self.train_lstm_alone
self.train_autoencoder = True and not self.train_lstm_alone
self.train_dis = False and not self.train_lstm_alone
self.load_models = load_models
self.last_best_result = 100
self.save_model_period = 501
self.save_sample_image_interval = 501
self.sample_image_cols = sample_image_cols
self.sample_image_rows = sample_image_rows
self.generator_test = self.dataset_ctrl.get_next_batch(task=['5002', '5001'], from_start_prob=0, human=False, train=False, sth_sth=False, camera='camera-1')
self.num_descs = self.dataset_ctrl.num_all_objects_describtors + 1
self.num_objects = self.dataset_ctrl.num_all_objects + 1
images_att, images, _, _, _ , _, _, _, objects, objs_one_hot, descriptions, descs_one_hot = next(self.generator_test)
images = images[:, 0]
images_att = images_att[:, 0]
images = np.reshape(images, (-1, self.num_channels, self.image_size, self.image_size))
images_att = np.reshape(images_att, (-1, self.num_channels, self.image_size, self.image_size))
# Preparing a set of sample images to test the network on and save the reconstructions etc
# Take a look at the function save_sample_images
sample_size = self.sample_image_cols * self.sample_image_rows
self.sample_images = np.asarray(images[:sample_size], dtype=np.float32)
self.sample_images_att = np.asarray(images_att[:sample_size], dtype=np.float32)
self.sample_objs_one_hot = np.asarray(objs_one_hot[:sample_size], dtype=np.float32)
self.sample_descs_one_hot = np.asarray(descs_one_hot[:sample_size], dtype=np.float32)
objects = np.asarray(objects[:sample_size], dtype=np.float32)
descriptions = np.asarray(descriptions[:sample_size], dtype=np.float32)
# target objects code ... take a look at the dictionary in the DatasetController_morph class
print('objects invloved code:')
print np.squeeze(objects)[:sample_size]
print np.squeeze(descriptions)[:sample_size]
self.generator = self.dataset_ctrl.get_next_batch(task=['5002', '5001'], from_start_prob=0, human=False, sth_sth=False, camera='camera-1')
self.enc_model = autoencoders.tower.Encoder_text_tower(density=8, size=image_size, latent_size=latent_size, channel=self.num_channels, num_objects=self.num_objects - 1, num_descriptions=self.num_descs - 1)
self.gen_model = autoencoders.tower.Generator_text(density=8, size=image_size, latent_size=latent_size, channel=self.num_channels * 2, num_objects=self.num_objects - 1, num_descriptions=self.num_descs - 1)
self.dis_model = autoencoders.tower.Discriminator_texual(density=8, size=image_size, channel=self.num_channels,
num_objects=self.num_objects, num_descriptions=self.num_descs)
self.mdn_model = RobotController(self.latent_size + self.num_objects + self.num_descs - 2, hidden_dimension, output_size, num_mixture, auto_regressive=False)
self.enc_models = [self.enc_model]
self.gen_models = [self.gen_model]
self.dis_models = [self.dis_model]
self.mdn_models = [self.mdn_model]
self.learning_rate = 0.0001
self.WeightDecay = 0.00001
self.optimizer_enc = optimizers.Adam(alpha=self.learning_rate, beta1=0.9)
self.optimizer_enc.setup(self.enc_models[0])
self.optimizer_enc.add_hook(chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_gen = optimizers.Adam(alpha=self.learning_rate, beta1=0.9)
self.optimizer_gen.setup(self.gen_models[0])
self.optimizer_gen.add_hook(chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_dis = optimizers.Adam(alpha=self.learning_rate, beta1=0.9)
self.optimizer_dis.setup(self.dis_models[0])
self.optimizer_dis.add_hook(chainer.optimizer.WeightDecay(self.WeightDecay))
self.optimizer_mdn = optimizers.Adam(alpha=self.learning_rate, beta1=0.9)
self.optimizer_mdn.setup(self.mdn_models[0])
self.optimizer_mdn.add_hook(chainer.optimizer.WeightDecay(0.00001))
for i in range(GPU.num_gpus - 1):
self.enc_models.append(copy.deepcopy(self.enc_model))
self.gen_models.append(copy.deepcopy(self.gen_model))
self.dis_models.append(copy.deepcopy(self.dis_model))
self.mdn_models.append(copy.deepcopy(self.mdn_model))
self.batch_gpu_threads = [None] * GPU.num_gpus
if self.load_models:
self.load_model()
self.to_gpu()