def __getitem__(self, idx):
curr_img_path = self.lines[idx]
curr_img_path = curr_img_path.rstrip()
img = Image.open(curr_img_path)
img = img.convert('RGB')
annotation_file = curr_img_path[:-3] + "txt"
boxes = []
labels = []
with open(annotation_file, "r") as f:
anno_lines = f.readlines()
for l in anno_lines:
c, x, y, w, h = l.split(" ")
boxes.append(yolo2pascalvoc(img.width, img.height, float(x), float(y), float(w), float(h)))
labels.append(int(c) + 1)
difficulties = torch.ByteTensor([0 for i in range(len(labels))])
boxes = torch.FloatTensor(boxes)
labels = torch.tensor(labels, dtype=torch.int64)
image, boxes, labels, difficulties = transform(img, boxes, labels, difficulties, split=self.split)
return image, boxes, labels, difficulties
def __getitem__(self, i):
# Read image
image = Image.open(self.images[i], mode='r')
image = image.convert('RGB')
# Read objects in this image (bounding boxes, labels, difficulties)
objects = self.objects[i]
boxes = torch.FloatTensor(objects['boxes']) # (n_objects, 4)
labels = torch.LongTensor(objects['labels']) # (n_objects)
difficulties = torch.ByteTensor(objects['difficulties']) # (n_objects)
# Discard difficult objects, if desired
if not self.keep_difficult:
boxes = boxes[1 - difficulties]
labels = labels[1 - difficulties]
difficulties = difficulties[1 - difficulties]
# Apply transformations
image, boxes, labels, difficulties = transform(image,
boxes,
labels,
difficulties,
split=self.split)
return image, boxes, labels, difficulties
def get(self, day=0):
day = self.how_many_files(day)
X = [None for _ in range(len(self.nfiles))]
for i in range(len(self.nfiles)):
X[i] = transform(
np.array(self.file.get(day + '/' + self.nfiles[i])))
return X
def __call__(self, img):
ori_im = img.copy()
data = {'image': img}
data = transform(data, self.preprocess_op)
img, shape_list = data
if img is None:
return None, 0
img = np.expand_dims(img, axis=0)
shape_list = np.expand_dims(shape_list, axis=0)
img = img.copy()
starttime = time.time()
self.input_tensor.copy_from_cpu(img)
self.predictor.run()
outputs = []
for output_tensor in self.output_tensors:
output = output_tensor.copy_to_cpu()
outputs.append(output)
preds = {}
preds['maps'] = outputs[0]
post_result = self.postprocess_op(preds, shape_list)
dt_boxes = post_result[0]['points']
dt_boxes = self.filter_tag_det_res(dt_boxes, ori_im.shape)
elapse = time.time() - starttime
return dt_boxes, elapse
def pixar():
try:
print(request.files)
img_file = request.files["file"]
filename = img_file.filename
filename = rename(filename)
if os.path.exists("./images") == False:
os.mkdir("./images")
img_file.save(os.path.join("./images", filename))
except Exception as error:
print(error)
return abort(400)
else:
res_path = transform(model, filename, delete_input=True)
if res_path == "-1":
return abort(404)
with open(res_path, "rb") as file:
encoded = base64.b64encode(file.read())
return encoded
def demon_geometry(self, image1, image2):
#transform and yield
image1_new = uts.transform(image1.copy(),
height=self.params['size'][0],
width=self.params['size'][1])
image2_new = uts.transform(image2.copy(),
height=self.params['size'][0],
width=self.params['size'][1])
self.intrinsic = self.params['intrinsic']
# down sample
test_data_bs = [(image1_new, image2_new)]
flow, depth_inv, normal, rotation, translation = paddle.infer(
output=self.topo_bs,
parameters=self.parameters_bs,
input=test_data_bs,
feeding=self.feeding_bs);
for i in range(3):
test_data_it = [(image1_new, image2_new, self.intrinsic,
rotation, translation, depth_inv, normal)]
flow, depth_inv, normal, rotation, translation = paddle.infer(
output=self.topo_it,
parameters=self.parameters_it,
input=test_data_it,
feeding=self.feeding_it);
test_data_re = [(image1_new, image2_new, depth_inv)]
depth = paddle.infer(output=self.topo_re,
parameters=self.parameters_re,
input=test_data_re,
feeding=self.feeding_re);
layer_names = [self.outputs_it['flow'].name,
self.outputs_it['normal'].name,
self.outputs_re['depth_0'].name]
height_list = [self.my_g_layer_map[x].height for x in layer_names]
width_list = [self.my_g_layer_map[x].width for x in layer_names]
flow, normal, depth = uts.vec2img(inputs=[flow, normal, depth],
height=height_list,
width=width_list)
motion = np.concatenate([rotation, translation])
return flow, normal, depth, motion
def get_one_sample(self,sample):
file_basename_image, file_basename_label, label =sample
image_path = os.path.join(self.data_dir, file_basename_image)
label_path = os.path.join(self.data_dir, file_basename_label)
image = self.read_data(image_path)
label_pixel = self.read_data(label_path)
label_pixel = self.label_preprocess(label_pixel)
if not self.with_RGB:
image = (np.array(image[:, :, np.newaxis]))
label_pixel = (np.array(label_pixel[:, :, np.newaxis]))
image = utils.transform(image)
return image, label_pixel, int(label), file_basename_image
def generate_sample_face(image,
landmarks,
detector,
input_resolution=256,
output_resolution=64):
num_landmarks = len(landmarks)
detected_faces = utils.get_face_bbox(image, detector)
outputs = list()
if len(detected_faces) > 0:
for i, rect in enumerate(detected_faces):
center = [(rect.left() + rect.right()) / 2,
(rect.top() + rect.bottom()) / 2]
center[1] = center[1] - (rect.bottom() - rect.top()) * 0.12
# scale = (rect.right() - rect.left() +
# rect.bottom() - rect.top()) / 195.0
scale = 2.0
cropped_image = utils.crop(image,
center,
scale,
resolution=input_resolution)
heatmaps = np.zeros(
(output_resolution, output_resolution, num_landmarks))
transformed_landmarks = []
for j in range(num_landmarks):
ldmk = utils.transform(landmarks[j] + 1,
center,
scale,
resolution=output_resolution)
transformed_landmarks.append(ldmk)
tmp = utils.draw_gaussian(heatmaps[:, :, j], ldmk, 1)
heatmaps[:, :, j] = tmp
outputs.append({
'image': cropped_image / 255,
'heatmaps': heatmaps,
'center': center,
'scale': scale,
'pts': transformed_landmarks
})
return outputs
def post_process(heatmaps, center, scale):
'''
:param heatmaps: ndarray shape{1, resolution, resolution, num_facial_landmarks}
:param center: cropped center
:param scale: scale factor
:return: landmarks of cropped image and original image
'''
resolution_y, resolution_x = heatmaps.shape[1], heatmaps.shape[2]
num_landmarks = heatmaps.shape[3]
landmarks = []
for i in range(num_landmarks):
heatmap = heatmaps[0, :, :, i]
indx = np.argmax(heatmap)
coord = np.unravel_index(indx, (resolution_y, resolution_x))
coord = [coord[1], coord[0]]
landmarks.append(list(coord))
landmarks_origins = []
for i in range(num_landmarks):
tmp = utils.transform(
np.asarray(landmarks[i]) + 1, center, scale, resolution_x, True)
landmarks_origins.append(tmp)
return landmarks, landmarks_origins
def _numerical_transform(self, numercial_columns_for_transform):
for col, way in numercial_columns_for_transform:
self.numerical_col.append(col + '_' + way)
self.dataset = self.dataset.map(
lambda row: utils.transform(row, col, way),
num_parallel_calls=config.NUM_PARALLEL)
from adapter import Adapt
from attention import NeuralModel
from postprocess import PostProcess
from preprocess import preprocess
from recommend import CFUtil
from utils.utils import transform
if __name__ == "__main__":
np.random.seed(100)
path = os.path.abspath('.')
pst = PostProcess(path)
samples, users, movies = preprocess()
samples = samples[0:20000]
users = transform(users)
movies = transform(movies)
# pst.saveSamples(samples, 'samples.csv')
# pst.saveReviews(users, 'users.csv')
# pst.saveReviews(movies, 'movies.csv')
# load data
# samples = pst.loadSamples('samples.csv')
# users = pst.loadReviews('users.csv')
# movies = pst.loadReviews('movies.csv')
# tuning
epoch = 10
l2 = 0.01
uhid = 128
def test_demo():
# PaddlePaddle init
paddle.init(use_gpu=True, gpu_id=FLAGS.gpu_id)
params = sun3d.set_params()
inputs = d_net.get_demon_inputs(params)
params['stage'] = 5
# Add neural network config
outputs, out_field = d_net.get_demon_outputs(inputs, params, ext_inputs=None)
parameters, topo = paddle.parameters.create(outputs[out_field])
# Read image pair 1, 2 flow
for scene_name in params['train_scene'][1:]:
image_list = preprocess_util.list_files(
params['flow_path'] + scene_name + '/flow/')
image2depth = sun3d.get_image_depth_matching(scene_name)
for pair_name in image_list[0:2]:
image1, image2, flow_gt, depth1_gt, normal1_gt = \
sun3d.load_image_pair(scene_name, pair_name, image2depth)
image1_new = uts.transform(image1.copy(),
height=params['size'][0],
width=params['size'][1])
image2_new = uts.transform(image2.copy(),
height=params['size'][0],
width=params['size'][1])
intrinsic = np.array([0.89115971, 1.18821287, 0.5, 0.5])
test_data = [(image1_new, image2_new, intrinsic)]
depth_name = 'depth' if params['stage'] < 5 else 'depth_0'
out_fields = ['flow', depth_name, 'normal', 'rotation',
'translation']
# out_fields = ['flow']
# height_list = [cp.g_layer_map[outputs[x].name].height \
# for x in ['flow']]
# width_list = [cp.g_layer_map[outputs[x].name].width \
# for x in ['flow']]
output_list = [outputs[x] for x in out_fields]
flow, depth, normal, rotation, translation = paddle.infer(
output=topo,
parameters=parameters,
input=test_data,
feeding={'image1': 0,
'image2': 1,
'intrinsic': 2});
height_list = [cp.g_layer_map[outputs[x].name].height \
for x in ['flow', depth_name,'normal']]
width_list = [cp.g_layer_map[outputs[x].name].width \
for x in ['flow', depth_name,'normal']]
# flow = paddle.infer(output=output_list,
# parameters=parameters,
# input=test_data,
# feeding={'image1': 0,
# 'image2': 1,
# 'intrinsic': 2});
# flow = vec2img(inputs=[flow],
# height=height_list,
# width=width_list)
# uts.plot_images(OrderedDict([('image1',image1),
# ('image2',image2),
# ('flow',flow),
# ('flow_gt',flow_gt)]),
# layout=[4,2])
flow, depth, normal = vec2img(inputs=[flow, depth, normal],
height=height_list,
width=width_list)
# visualize depth in 3D
# image1_down = cv2.resize(image1,
# (depth.shape[1], depth.shape[0]))
# visualize_prediction(
# depth=depth,
# image=np.uint8(image1_down.transpose([2, 0, 1])),
# rotation=rotation,
# translation=translation)
uts.plot_images(OrderedDict([('image1',image1),
('image2',image2),
('flow',flow),
('flow_gt',flow_gt),
('depth', depth),
('depth_gt', depth1_gt)]),
# ('normal', (normal + 1.0)/2.),
# ('normal_gt', (normal1_gt + 1.0)/2)]),
layout=[4,2])
def test_refine_net(dataset='sun3d', split='train', vis=False):
paddle.init(use_gpu=True, gpu_id=FLAGS.gpu_id)
params = sun3d.set_params()
part, part_id = [int(x) for x in FLAGS.part.split(',')]
test_ids = partition(len(params[split + '_scene']), part, part_id)
rate = 0.05
is_inverse = False
depth_name = 'depth_inv' if is_inverse else 'depth'
process_scene_names = [params[split + '_scene'][x] for x in test_ids]
inputs = u_net.get_inputs(params)
outputs = u_net.refine_net(inputs, params)
parameters, topo = paddle.parameters.create(outputs[depth_name])
print('load parameters {}'.format(FLAGS.model))
with gzip.open(FLAGS.model, 'r') as f:
parameters = paddle.parameters.Parameters.from_tar(f)
feeding = {'image1': 0, 'depth': 1}
for scene_name in process_scene_names:
id_img2depth = sun3d.get_image_depth_matching(scene_name)
upsample_output_path = params['flow_path'] + scene_name + \
'/pair_depth/' + str(rate) + '/'
prefix_len = len(upsample_output_path)
image_list = preprocess_util.list_files(upsample_output_path)
for pair_name in image_list:
print pair_name
pair_image_name = pair_name.split('/')[-1]
outfile = upsample_output_path + pair_image_name[:-4] + '.npy'
depth_net = np.load(outfile)
depth_net_in = depth_net.flatten()
if is_inverse:
depth_net_in = uts_3d.inverse_depth(depth_net)
image_name1, _ = pair_image_name.split('_')
image_path1 = params['data_path'] + scene_name + \
'/image/' + image_name1 + '.jpg'
depth_path1 = params['data_path'] + scene_name + '/depth/' + \
id_img2depth[image_name1] + '.png'
image1 = cv2.imread(image_path1)
depth1 = uts.read_depth(depth_path1)
image1_new = uts.transform(image1.copy(),
height=params['size'][0],
width=params['size'][1])
test_data = [(
image1_new,
depth_net_in,
)]
print 'forward'
depth_out = paddle.infer(output=topo,
parameters=parameters,
input=test_data,
feeding=feeding)
if is_inverse:
depth_out = uts_3d.inverse_depth(depth_out)
depth = uts.vec2img(inputs=depth_out,
height=params['size'][0],
width=params['size'][1])
if vis:
uts.plot_images(OrderedDict([('image', image1),
('depth1', depth1),
('depth_net', depth_net),
('depth', depth)]),
layout=[4, 2])
def test_demo():
# PaddlePaddle init
paddle.init(use_gpu=True, gpu_id=FLAGS.gpu_id)
params = sun3d.set_params()
inputs = d_net.get_demon_inputs(params)
# Add neural network config
outputs_bs = d_net.bootstrap_net(inputs, params)
outputs_it = d_net.iterative_net(inputs, params)
outputs_re = d_net.refine_net(inputs, params)
out_fields = ['flow', 'depth_inv', 'normal', 'rotation', 'translation']
my_g_layer_map = {}
parameters_bs, topo_bs = paddle.parameters.create(
[outputs_bs[x] for x in out_fields])
my_g_layer_map.update(cp.g_layer_map)
parameters_it, topo_it = paddle.parameters.create(
[outputs_it[x] for x in out_fields])
my_g_layer_map.update(cp.g_layer_map)
parameters_re, topo_re = paddle.parameters.create(outputs_re['depth_0'])
my_g_layer_map.update(cp.g_layer_map)
print('load parameters')
with gzip.open(FLAGS.model, 'r') as f:
parameters_init = paddle.parameters.Parameters.from_tar(f)
for name in parameters_bs.names():
parameters_bs.set(name, parameters_init.get(name))
for name in parameters_it.names():
parameters_it.set(name, parameters_init.get(name))
for name in parameters_re.names():
parameters_re.set(name, parameters_init.get(name))
# Read image pair 1, 2 flow
for scene_name in params['train_scene'][1:]:
image_list = preprocess_util.list_files(params['flow_path'] +
scene_name + '/flow/')
image2depth = sun3d.get_image_depth_matching(scene_name)
for pair_name in image_list[0:2]:
image1, image2, flow_gt, depth1_gt, normal1_gt = \
sun3d.load_image_pair(scene_name, pair_name, image2depth)
#transform and yield
image1_new = uts.transform(image1.copy(),
height=params['size'][0],
width=params['size'][1])
image2_new = uts.transform(image2.copy(),
height=params['size'][0],
width=params['size'][1])
intrinsic = np.array([0.89115971, 1.18821287, 0.5, 0.5])
test_data_bs = [(image1_new, image2_new)]
feeding_bs = {'image1': 0, 'image2': 1}
flow, depth_inv, normal, rotation, translation = paddle.infer(
output=topo_bs,
parameters=parameters_bs,
input=test_data_bs,
feeding=feeding_bs)
for i in range(3):
test_data_it = [(image1_new, image2_new, intrinsic, rotation,
translation, depth_inv, normal)]
feeding_it = {
'image1': 0,
'image2': 1,
'intrinsic': 2,
'rotation': 3,
'translation': 4,
'depth_inv': 5,
'normal': 6
}
flow, depth_inv, normal, rotation, translation = paddle.infer(
output=topo_it,
parameters=parameters_it,
input=test_data_it,
feeding=feeding_it)
test_data_re = [(image1_new, image2_new, depth_inv)]
feeding_re = {'image1': 0, 'image2': 1, 'depth_inv': 2}
depth = paddle.infer(output=topo_re,
parameters=parameters_re,
input=test_data_re,
feeding=feeding_re)
layer_names = [
outputs_it['flow'].name, outputs_it['normal'].name,
outputs_re['depth_0'].name
]
height_list = [my_g_layer_map[x].height for x in layer_names]
width_list = [my_g_layer_map[x].width for x in layer_names]
flow, normal, depth = vec2img(inputs=[flow, normal, depth],
height=height_list,
width=width_list)
# visualize depth in 3D
# image1_down = cv2.resize(image1,
# (depth.shape[1], depth.shape[0]))
# visualize_prediction(
# depth=depth,
# image=np.uint8(image1_down.transpose([2, 0, 1])),
# rotation=rotation,
# translation=translation)
with open('./test/depth_gt.npy', 'wb') as f:
np.save(f, depth1_gt)
with open('./test/depth_res.npy', 'wb') as f:
np.save(f, depth)
uts.plot_images(OrderedDict([
('image1', image1), ('image2', image2), ('flow', flow),
('flow_gt', flow_gt), ('depth', depth),
('depth_gt', depth1_gt), ('normal', (normal + 1.0) / 2.),
('normal_gt', (normal1_gt + 1.0) / 2)
]),
layout=[4, 2])