def generate_targets(sphereH, network, layer, tilt, ks):
# prepare output directory
kernel_size = kernel_sizes[layer]
if kernel_size > ks:
sys.stder.write("Image size should be larger than receptive field.")
sys.stder.write("Ks: {0} -> {1}\n".format(ks, kernel_size))
ks = kernel_size
target_dir = os.path.join(DATA_ROOT,
"TargetSphereH{0}Ks{1}".format(sphereH, ks),
"{0}{1}".format(network, layer))
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
# Check if output exists
train_path = os.path.join(target_dir, 'tilt{0:03d}.train.pkl'.format(tilt))
build_train = not os.path.exists(train_path)
test_path = os.path.join(target_dir, 'tilt{0:03d}.test.pkl'.format(tilt))
build_test = not os.path.exists(test_path)
if not build_train and not build_test:
return
# prepare projection matrix
sphereW = sphereH * 2
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
imgW=ks)
P = projection.buildP(tilt)
train = []
test = []
net = load_network(layer=layer, network=network, silence=False)
samples = sample_pixels(tilt, sphereH=sphereH)
for path, pixels in samples.viewitems():
print os.path.basename(path)
img = cv2.imread(path)
if sphereH != img.shape[0]:
img = cv2.resize(img, (sphereW, sphereH))
if "mountain_climbing" in path:
if not build_test:
continue
targets = test
else:
if not build_train:
continue
targets = train
for x, y in pixels:
assert y == tilt
rimg = np.roll(img, sphereW / 2 - x, axis=1)
patch = projection.project(P, rimg)
val = forward_patch(patch, net, layer=layer)
target = (path, x, y, val.ravel().tolist())
targets.append(target)
if build_train:
dump_pkl(train, train_path)
if build_test:
dump_pkl(test, test_path)
def __init__(self, layer, sphereH=320, ks=640, fov=65.5):
sphereW = sphereH * 2
kernel_size = kernel_sizes[layer]
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
view_angle=fov,
imgW=ks)
net = load_network(layer=layer, network="faster-rcnn")
self.sphereH = sphereH
self.sphereW = sphereW
self.Ps = {}
self.projection = projection
self.layer = layer
self.net = net
def predict_scores(feature):
def forward(W, b, v):
vp = v.copy()
vp[v < 0.] = 0.
out = np.dot(W, vp)
out += b[:, None]
return out
net = load_network(layer="8", network="faster-rcnn")
W_fc7 = net.params['fc7'][0].data
b_fc7 = net.params['fc7'][1].data
W_cls = net.params['cls_score'][0].data
b_cls = net.params['cls_score'][1].data
fc7 = forward(W_fc7, b_fc7, feature)
score = forward(W_cls, b_cls, fc7)
return score
def __init__(self, layer, sphereH=320, ks=640, fov=65.5):
sphereW = sphereH *2
base = "1_1"
kernel_size = kernel_sizes[base]
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
view_angle=fov,
imgW=ks)
base_net = load_network(layer=base, network='faster-rcnn')
n_c = base_net.blobs['conv{}'.format(base)].shape[1]
stride = strides[ks][sphereH][layer]
self.sphereW = sphereW
self.sphereH = sphereH
self.projection = projection
self.base_net = base_net
self.base = base
self.n_c = n_c
self.Ps = {}
self.stride = stride
def __init__(self, layer, sphereH=320, ks=640, fov=65.5):
sphereW = sphereH * 2
bot = top_down[layer]
kernel_size = kernel_sizes[bot]
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
view_angle=fov,
imgW=ks)
base_net = load_network(layer=bot, network='faster-rcnn')
n_c = base_net.blobs['conv{}'.format(bot)].shape[1]
supports = {}
stride = strides[ks][sphereH][layer]
for y in xrange(sphereH):
crop_in, _ = rounded_rf(layer, y, sphereH, ks)
h, w = crop_in
n_w = (w - 1) / stride / 2
n_h = (h - 1) / stride / 2
hs = np.arange(-n_h * stride, n_h * stride + 1, step=stride)
ws = np.arange(-n_w * stride, n_w * stride + 1, step=stride)
Px, Py = np.meshgrid(hs, ws)
Py += y
supports[y] = (Px, Py)
self.supports = supports
self.n_c = n_c
self.sphereW = sphereW
self.sphereH = sphereH
self.ks = ks
self.layer = layer
self.bot = bot
self.base_net = base_net
self.projection = projection
self.Ps = {}
self.top_nets = {}
def generate_source(frame, layer, sphereH=320, ks=640, network='faster-rcnn'):
# collect frame information
frames = collect_frames()
frame_path = frames[frame]
frameId = get_frameId(frame_path)
# Prepare path / check existing file
kernel_size = kernel_sizes[layer]
if kernel_size > ks:
sys.stder.write("Image size should be larger than receptive field.")
sys.stder.write("Ks: {0} -> {1}\n".format(ks, kernel_size))
ks = kernel_size
src_dir = os.path.join(DATA_ROOT,
"SourceSphereH{0}Ks{1}".format(sphereH, ks),
"{0}{1}".format(network, layer))
if not os.path.isdir(src_dir):
os.makedirs(src_dir)
src_path = os.path.join(src_dir, "{}.h5".format(frameId))
if os.path.exists(src_path):
sys.stderr.write("{} exists.\n".format(src_path))
return
lockfile = "{}.lock".format(src_path)
if os.path.isfile(lockfile):
sys.stderr.write("{} is being generated by other process.\n".format(src_path))
return
open(lockfile, "a").close()
log_dir = os.path.join(LOG_ROOT, "SphereH{0}".format(sphereH),
"{0}{1}".format(network, layer))
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_path = os.path.join(log_dir, "{}.log".format(frameId))
log = Logger(log_path)
batch_size = 128
# load network
net = load_network(layer=layer, network=network)
in_shape = net.blobs['data'].shape
net.blobs['data'].reshape(batch_size, in_shape[1], in_shape[2], in_shape[3])
net.reshape()
# prepare projection matrix
sphereW = sphereH * 2
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
imgW=ks)
# Assume convolutional layer
layer_name = 'conv{}'.format(layer)
Nch = net.params[layer_name][0].data.shape[0]
src = np.zeros((sphereH, sphereW, Nch), dtype=np.float32)
log.write("Start computing convolution.\n")
img = cv2.imread(frame_path)
if sphereH != img.shape[0]:
img = cv2.resize(img, (sphereW, sphereH))
for y in range(sphereH):
P = projection.buildP(y)
log.write("Process the {:>3d}-th row\n".format(y))
log.flush()
for x_start in range(0, sphereW, batch_size):
for i in range(batch_size):
x = x_start + i
rimg = np.roll(img, projection.sphereW/2-x, axis=1)
patch = projection.project(P, rimg)
patch = patch - np.array([103.939, 116.779, 123.68])
patch = np.transpose(patch, (2,0,1))
net.blobs['data'].data[i,...] = patch
target = 'conv{}'.format(layer)
out = net.forward(blobs=[target])
vals = out[target].copy()
assert vals.shape[2] == 1
assert vals.shape[3] == 1
for i in range(batch_size):
x = x_start + i
src[y,x,:] = vals[i].ravel()
log.write("Write output to disk.\n")
with h5py.File(src_path, 'w') as hf:
hf.create_dataset(frameId, data=src,
compression="lzf",
shuffle=True,
chunks=(128, 128, Nch))
os.remove(lockfile)
def generate_source(path, tilt, layer, **kwargs):
sphereH = kwargs.get('sphereH', 320)
sphereW = sphereH * 2
ks = kwargs.get('ks', 640)
rf_size = kwargs.get('rf_size', 224)
dirs, frame = os.path.split(path)
#print(dirs)
#print(frame)
dirs = dirs.split("/")
src_dir = os.path.join(DATA_ROOT, "Pano_Superpoint", "train", layer)
if not os.path.isdir(src_dir):
os.makedirs(src_dir)
frameId = os.path.splitext(frame)[0]
src_path = os.path.join(src_dir, "{}.h5".format(frameId))
if os.path.exists(src_path):
sys.stderr.write("{} exists.\n".format(src_path))
return
lockfile = "{}.lock".format(src_path)
if os.path.isfile(lockfile):
sys.stderr.write(
"{} is being generated by other process.\n".format(src_path))
return
open(lockfile, "a").close()
log_dir = os.path.join(LOG_ROOT, "Pano_Superpoint", "train", layer)
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_path = os.path.join(log_dir, "{}.log".format(frameId))
log = Logger(log_path)
log.write("Generate output {}\n".format(src_path))
# prepare projection matrix and network
#layer = "5_3"
print("Funciona")
net = load_network(layer=layer, network='superpoint')
print("dejo de funcionar")
kernel_size = kernel_sizes[layer]
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
imgW=ks)
# Assume convolutional layer
layer_name = '{}'.format(layer)
Nch = net.params[layer_name][0].data.shape[0]
src = np.zeros((sphereH, sphereW, Nch), dtype=np.float32)
print(src.shape)
log.write("Start computing convolution.\n")
img = cv2.imread(path + '.jpg', cv2.COLOR_BGR2GRAY)
#print(img.shape)
if sphereH != img.shape[0]:
#print("entro")
img = cv2.resize(img, (sphereW, sphereH))
#print(img.shape)
img = img.reshape((320, 640, 1))
#print(img.shape)
for y in range(sphereH):
P = projection.buildP(y)
#print(P.shape)
log.write("Process the {:>3d}-th row\n".format(y))
log.flush()
for x in range(sphereW):
rimg = np.roll(img, int(projection.sphereW / 2 - x), axis=1)
#print(rimg.shape)
patch = projection.project(P, rimg)
#print(patch.shape)
val = forward_patch(patch, net, layer=layer)
#print(val.shape)
assert val.shape[2] == 1
assert val.shape[3] == 1
src[y, x, :] = val.ravel()
log.write("Write output to disk.\n")
#print(src)
with h5py.File(src_path, 'w') as hf:
hf.create_dataset(frameId,
data=src,
compression="lzf",
shuffle=True,
chunks=(128, 128, Nch))
os.remove(lockfile)
def generate_source(path, tilt, **kwargs):
sphereH = kwargs.get('sphereH', 320)
sphereW = sphereH * 2
ks = kwargs.get('ks', 640)
rf_size = kwargs.get('rf_size', 224)
dirs, frame = os.path.split(path)
dirs = dirs.split("/")
src_dir = os.path.join(DATA_ROOT, "Pano", "Rf{}".format(rf_size), dirs[-2],
"tilt{:03d}".format(tilt))
if not os.path.isdir(src_dir):
os.makedirs(src_dir)
frameId = os.path.splitext(frame)[0]
src_path = os.path.join(src_dir, "{}.h5".format(frameId))
if os.path.exists(src_path):
sys.stderr.write("{} exists.\n".format(src_path))
return
lockfile = "{}.lock".format(src_path)
if os.path.isfile(lockfile):
sys.stderr.write(
"{} is being generated by other process.\n".format(src_path))
return
open(lockfile, "a").close()
log_dir = os.path.join(LOG_ROOT, "Pano", "Rf{}".format(rf_size), dirs[-2],
"tilt{:03d}".format(tilt))
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_path = os.path.join(log_dir, "{}.log".format(frameId))
log = Logger(log_path)
log.write("Generate output {}\n".format(src_path))
# prepare projection matrix and network
layer = "1_1"
net = load_network(layer=layer, network='faster-rcnn')
kernel_size = kernel_sizes[layer]
projection = SphereProjection(kernel_size=kernel_size,
sphereW=sphereW,
sphereH=sphereH,
imgW=ks)
# Assume convolutional layer
layer_name = 'conv{}'.format(layer)
Nch = net.params[layer_name][0].data.shape[0]
src = np.zeros((sphereH, sphereW, Nch), dtype=np.float32)
log.write("Start computing convolution.\n")
img = cv2.imread(path)
if sphereH != img.shape[0]:
img = cv2.resize(img, (sphereW, sphereH))
for y in xrange(sphereH):
P = projection.buildP(y)
log.write("Process the {:>3d}-th row\n".format(y))
log.flush()
for x in xrange(sphereW):
rimg = np.roll(img, projection.sphereW / 2 - x, axis=1)
patch = projection.project(P, rimg)
val = forward_patch(patch, net, layer=layer)
assert val.shape[2] == 1
assert val.shape[3] == 1
src[y, x, :] = val.ravel()
log.write("Write output to disk.\n")
with h5py.File(src_path, 'w') as hf:
hf.create_dataset(frameId,
data=src,
compression="lzf",
shuffle=True,
chunks=(128, 128, Nch))
os.remove(lockfile)
def generate_proposals(imgs, tilt, method, **kwargs):
def img_proposals(path):
frameId = os.path.splitext(os.path.basename(path))[0]
print frameId
img = cv2.imread(path)
if sphereH != img.shape[0]:
img = cv2.resize(img, (sphereW, sphereH))
data = np.zeros((Px.shape[0], Px.shape[1], Nch))
xs = []
ys = []
idx = []
for i in xrange(Px.shape[0]):
for j in xrange(Px.shape[1]):
x = Px[i, j]
y = Py[i, j]
x = int(round(x))
y = int(round(y))
xs.append(x)
ys.append(y)
idx.append((i, j))
if method == "sphconv":
frameId = os.path.splitext(os.path.basename(path))[0]
base_path = os.path.join(base_dir, "{}.h5".format(frameId))
vals = extractor.extract_convs(img, xs, ys, conv_path=base_path)
else:
vals = extractor.extract_convs(img, xs, ys)
for (i, j), val in zip(idx, vals):
data[i, j, :] = val
data = np.transpose(data, (2, 0, 1))
rpn_net.blobs['data'].data[0] = data
targets = ['rpn_cls_prob_reshape', 'rpn_bbox_pred']
out = rpn_net.forward(blobs=targets)
scores = out['rpn_cls_prob_reshape']
scores = scores[:, num_anchors:, :, :].reshape((num_anchors, 1))
bbox_deltas = out['rpn_bbox_pred']
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
proposals = bbox_transform_inv(anchors, bbox_deltas)
proposals = clip_boxes(proposals, ks=ks)
proposals = np.hstack([proposals, scores])
return proposals
sphereH = kwargs.get("sphereH", 640)
sphereW = sphereH * 2
fov = kwargs.get("view_angle", 65.5)
ks = kwargs.get("ks", 640)
rf = SphereCoordinates(kernel_size=224,
sphereW=sphereW,
sphereH=sphereH,
view_angle=fov,
imgW=ks)
Px, Py = rf.generate_grid(tilt)
Px = Px[8::16, 8::16]
Py = Py[8::16, 8::16]
Px = Px[5:8, 5:8]
Py = Py[5:8, 5:8]
anchor_scales = np.array((8, 16, 32))
anchors = generate_anchors(scales=anchor_scales)
anchors += ks / 2
num_anchors = anchors.shape[0]
bot = "5_3"
if method == "exact":
extractor = ExactProjection(layer=bot, sphereH=sphereH, ks=ks, fov=fov)
elif method == "optconv":
extractor = SphericalConvolution(layer=bot,
sphereH=sphereH,
ks=ks,
fov=fov)
elif method == "sphconv":
extractor = SphConv(layer=bot, sphereH=sphereH, ks=ks, fov=fov)
root = os.path.join(DATA_ROOT, "PanoTarget")
rf_size = kwargs["rf_size"]
voc = kwargs["voc"]
base_dir = os.path.join(root, "Rf{}".format(rf_size),
"VOC{}".format(voc), "tilt{:03d}".format(tilt))
else:
raise ValueError("Not implement")
rpn_net = load_network(layer="RPN", network='faster-rcnn')
Nch = rpn_net.blobs['data'].shape[1]
proposals = {path: img_proposals(path) for path in imgs}
return proposals
def backproject_convolution(imgs, tilt, method, **kwargs):
def evaluate_fc6(path):
frameId = os.path.splitext(os.path.basename(path))[0]
print frameId
img = cv2.imread(path)
if sphereH != img.shape[0]:
img = cv2.resize(img, (sphereW, sphereH))
data = np.zeros((kh, kw, Nch))
xs = []
ys = []
idx = []
for i in xrange(kh):
for j in xrange(kw):
x = Px[i, j]
y = Py[i, j]
x = int(round(x))
y = int(round(y))
xs.append(x)
ys.append(y)
idx.append((i, j))
if method == "sphconv":
base_path = os.path.join(base_dir, "{}.h5".format(frameId))
vals = extractor.extract_convs(img, xs, ys, conv_path=base_path)
else:
vals = extractor.extract_convs(img, xs, ys)
for (i, j), val in zip(idx, vals):
data[i, j, :] = val
data = np.transpose(data, (2, 0, 1))
fc_net.blobs['data'].data[0] = data
target = 'fc6'
out = fc_net.forward(blobs=[target])
fc6 = out[target].copy()
return fc6
sphereH = kwargs.get("sphereH", 640)
sphereW = sphereH * 2
fov = kwargs.get("view_angle", 65.5)
ks = kwargs.get("ks", 640)
rf = SphereCoordinates(kernel_size=224,
sphereW=sphereW,
sphereH=sphereH,
view_angle=fov,
imgW=ks)
Px, Py = rf.generate_grid(tilt)
Px = Px[8::16, 8::16]
Py = Py[8::16, 8::16]
bot = "5_3"
if method == "exact":
extractor = ExactProjection(layer=bot, sphereH=sphereH, ks=ks, fov=fov)
elif method == "optconv":
extractor = SphericalConvolution(layer=bot,
sphereH=sphereH,
ks=ks,
fov=fov)
elif method == "sphconv":
extractor = SphConv(layer=bot, sphereH=sphereH, ks=ks, fov=fov)
root = os.path.join(DATA_ROOT, "PanoTarget")
rf_size = kwargs["rf_size"]
voc = kwargs["voc"]
base_dir = os.path.join(root, "Rf{}".format(rf_size),
"VOC{}".format(voc), "tilt{:03d}".format(tilt))
else:
raise ValueError("Not implement")
fc_net = load_network(layer="FC6", network='faster-rcnn')
_, Nch, kh, kw = fc_net.blobs['data'].shape
fc6 = np.vstack([evaluate_fc6(path) for path in imgs]).transpose()
return fc6