def update(self):
if self.manager.state('raceStatus') is None:
startCropped = cv2.Canny(
util.crop(self.window,
np.hstack(self.startRect) + (-5, -5, 10, 10)), 350,
200)
val = np.max(
scipy.signal.correlate2d(startCropped,
goTemplate,
mode="valid"))
if val > 200:
self.manager.state('raceStatus', "started")
else:
cropped = util.crop(self.window,
np.hstack(self.finishRect) + (-5, -5, 10, 10))
threshold = 25
# three channels
numPixels = cropped.size / 3
numOrangeish = np.sum(
np.logical_and(
np.all(cropped > (self.color - threshold), axis=2),
np.all(cropped < (self.color + threshold), axis=2)))
if numOrangeish / float(numPixels) > .15:
finishCropped = cv2.Canny(cropped, 350, 200)
vals = scipy.signal.correlate2d(finishCropped,
finishTemplate,
mode="valid")
if (np.max(vals) > 210):
if self.manager.state('raceStatus') != 'finished':
self.manager.state('raceStatus', 'finished')
def update(self):
setupCrop = util.crop(self.window, np.hstack(self.setupRect))
if (np.mean(setupCrop) > self.setupThreshold):
self.lastMode = "setup"
self.manager.state("mode", "setup")
return
else:
# must be either overview, racing, or endSequence
if self.lastMode == "overview":
# if we just came from the overview
self.lastMode = "racing"
self.manager.state("mode", "racing")
return
if self.lastMode == None:
# if we just started the video. usually debugging
self.lastMode = "racing"
self.manager.state("mode", "racing")
return
else:
# last mode either racing or setup
# could be either overview or endSequence
isOverview = False
overviewCrop = util.crop(self.window, np.hstack(self.overviewRect))
gray = cv2.cvtColor(overviewCrop, cv2.COLOR_BGR2GRAY)
t = np.zeros(overviewCrop.shape)
if (np.sum(gray < 2) > 200):
isOverview = True
if isOverview:
self.lastMode = "overview"
self.manager.state("mode", "overview")
else:
self.lastMode = "endSequence"
self.manager.state("mode", "endSequence")
return
def main():
df = to_df(RAW_DATA_FILE)
df, item_cnt, feature_size = remap(df)
user_df, item_df = gen_user_item_group(df, item_cnt, feature_size)
# get_stat(user_df, item_df, STAT_PKL_PATH)
# plt_stat(STAT_PKL_PATH, FIG_PATH)
gen_dataset(user_df, item_df, item_cnt, feature_size, DATASET_PKL)
fin = open('../data/taobao/dataset.pkl', 'r')
trainset = pkl.load(fin)
testset = pkl.load(fin)
feature_size = pkl.load(fin)
fout_crop = open('../data/taobao/dataset_crop.pkl', 'wb')
fout_fp = open('../data/taobao/dataset_hpmn.pkl', 'wb')
train_crop = []
test_crop = []
train_fp = []
test_fp = []
for seq in trainset:
train_crop.append(crop(seq, 44, 300, 4, 8, 36, 3))
train_fp.append(front_padding(seq, 300, 4, 36, 3))
del trainset
for seq in testset:
test_crop.append(crop(seq, 44, 300, 4, 8, 36, 3))
test_fp.append(front_padding(seq, 300, 4, 36, 3))
pkl.dump(train_crop, fout_crop)
pkl.dump(test_crop, fout_crop)
pkl.dump(feature_size, fout_crop)
pkl.dump(train_fp, fout_fp)
pkl.dump(test_fp, fout_fp)
pkl.dump(feature_size, fout_fp)
fin.close()
def main():
df = aggregator(REVIEW_FILE, META_FILE)
df, item_cnt, feature_size = remap(df)
user_df, item_df = gen_user_item_group(df, item_cnt, feature_size)
# plt_stat(SAVE_PKL_PATH, FIG_PATH)
gen_dataset(SAVE_PKL_PATH, DATASET_PKL_PATH)
fin = open('../data/amazon/dataset.pkl', 'r')
trainset = pkl.load(fin)
testset = pkl.load(fin)
feature_size = pkl.load(fin)
fout_crop = open('../data/amazon/dataset_crop.pkl', 'wb')
fout_fp = open('../data/amazon/dataset_hpmn.pkl', 'wb')
train_crop = []
test_crop = []
train_fp = []
test_fp = []
for seq in trainset:
train_crop.append(crop(seq, 10, 100, 3, 10, 100, 2))
train_fp.append(front_padding(seq, 100, 3, 100, 2))
del trainset
for seq in testset:
test_crop.append(crop(seq, 10, 100, 3, 10, 100, 2))
test_fp.append(front_padding(seq, 100, 3, 100, 2))
pkl.dump(train_crop, fout_crop)
pkl.dump(test_crop, fout_crop)
pkl.dump(feature_size, fout_crop)
pkl.dump(train_fp, fout_fp)
pkl.dump(test_fp, fout_fp)
pkl.dump(feature_size, fout_fp)
fin.close()
fout_crop.close()
fout_fp.close()
def update(self):
gray = cv2.cvtColor(self.window, cv2.COLOR_BGR2GRAY)
win = gray
vals = []
crop = util.crop(
win,
np.hstack((self.topLeft[self.direction], size)) + (-5, -5, 10, 10))
area = cv2.Canny(crop, 400, 300)
for index, template in enumerate(templates):
val = np.max(scipy.signal.correlate2d(
area, template, mode="valid")) / float(np.sum(template))
vals.append(val)
#if self.manager.id == 2:
# print vals
self.lap = np.argmax(vals) + 1
if self.lastLap == self.lap:
self.verified += 1
else:
self.verified = 0
self.lastLap = self.lap
if self.verified > 2:
self.manager.state('lap', self.lap)
def warp_patch(self, ng_path, z, bbox, res_mip_range, mip):
influence_bbox = deepcopy(bbox)
influence_bbox.uncrop(self.max_displacement, mip=0)
agg_flow = influence_bbox.identity(mip=mip)
agg_flow = np.expand_dims(agg_flow, axis=0)
agg_res = data_handler.get_aggregate_rel_flow(
z, influence_bbox, res_mip_range, mip, self.process_low_mip,
self.process_high_mip, self.x_res_ng_paths, self.y_res_ng_paths)
agg_flow += agg_res
raw_data = data_handler.get_image_data(ng_path, z, influence_bbox, mip)
#no need to warp if flow is identity
#warp introduces noise
if not influence_bbox.is_identity_flow(agg_flow, mip=mip):
warped = warp(raw_data, agg_flow)
else:
#print ("not warping")
warped = raw_data[0]
mip_disp = int(self.max_displacement / 2**mip)
cropped = crop(warped, mip_disp)
result = data_handler.preprocess_data(cropped * 256)
#preprocess divides by 256 and puts it into right dimensions
#this data range is good already, so mult by 256
data_handler.save_image_patch(self.dst_ng_path, result, z, bbox, mip)
def pre_process(self, data, image):
self.detection = data.pop("detection", True)
boundingbox = data.get("boundingbox", None)
if boundingbox:
bbox = list(map(int, boundingbox.split(",")))
image = util.crop(image, bbox)
return image
def instance_segmentation_api(img_path, threshold=0.5):
masks = get_prediction(img_path, threshold)
img = cv2.imread(img_path)
mask1 = None
mask2 = None
mask_all = None
persons1 = None
persons2 = None
for i in range(len(masks)):
mask_ = get_masks(masks[i])
if (mask1 is None):
mask1 = mask_
elif (mask2 is None):
mask2 = mask_
else:
if (random.random() < 0.5):
mask1 = cv2.addWeighted(mask1, 1, mask_, 1, 0)
else:
mask2 = cv2.addWeighted(mask2, 1, mask_, 1, 0)
if (mask1 is not None and mask2 is not None):
mask_all = cv2.addWeighted(mask1, 1, mask2, 1, 0)
elif (mask2 is None):
mask_all = mask1
if (mask_all is None):
output = img
else:
output = cv2.addWeighted(img, 1, mask_all, 1, 0)
if (mask1 is not None):
persons1 = cv2.bitwise_and(img, mask1)
if (mask2 is not None):
persons2 = cv2.bitwise_and(img, mask2)
output = crop(output)
if (mask_all is not None):
mask_all = crop(mask_all)
if (persons1 is not None):
persons1 = crop(persons1)
if (persons2 is not None):
persons2 = crop(persons2)
# cv2.imwrite('input.png', output)
# cv2.imwrite('mask.png', get_transparent_img(mask))
# cv2.imwrite('persons.png', get_transparent_img(persons))
return output, mask_all, persons1, persons2
def train(m, p=None):
d = DataLoader(BSD3000(),
o.batch_size,
num_workers=o.num_workers,
shuffle=True)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
mse = torch.nn.MSELoss()
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
k = k.flip(1, 2)
x = y
if p:
with torch.no_grad():
x = p([x, y, k, s])
optimizer.zero_grad()
out = m([x, y, k, s])
log("out", out)
out = crop(out, k)
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
if num > (o.epoch * iter_num - 4):
# if num % 6 == 1:
show(torch.cat((crop(y, k)[0, 0], g[0, 0], out[0, 0]), 1),
# save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot([i + 1 for i in range(len(losss))], losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
def inpainting_api(image, mask):
FLAGS = ng.Config('inpaint.yml')
tf.reset_default_graph()
model = InpaintCAModel()
# image = cv2.imread(img_path)
# mask = cv2.imread(mask_path)
# cv2.imwrite('new.png', image - mask)
# mask = cv2.resize(mask, (0,0), fx=0.5, fy=0.5)
assert image.shape == mask.shape
image = crop(image)
mask = crop(mask)
print('Shape of image: {}'.format(image.shape))
image = np.expand_dims(image, 0)
mask = np.expand_dims(mask, 0)
input_image = np.concatenate([image, mask], axis=2)
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
input_image = tf.constant(input_image, dtype=tf.float32)
output = model.build_server_graph(FLAGS, input_image)
output = (output + 1.) * 127.5
output = tf.reverse(output, [-1])
output = tf.saturate_cast(output, tf.uint8)
# load pretrained model
vars_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
assign_ops = []
for var in vars_list:
vname = var.name
from_name = vname
var_value = tf.contrib.framework.load_variable(
'./model_logs/inpaint', from_name)
assign_ops.append(tf.assign(var, var_value))
sess.run(assign_ops)
print('Model loaded.')
result = sess.run(output)
return result[0][:, :, ::-1]
def update(self):
setupCrop = util.crop(self.window, np.hstack(self.setupRect))
if (np.mean(setupCrop) > self.setupThreshold):
self.lastMode = "setup"
self.manager.state("mode", "setup")
return
else:
# must be either overview, racing, or endSequence
if self.lastMode == "overview":
# if we just came from the overview
self.lastMode = "racing"
self.manager.state("mode", "racing")
return
if self.lastMode == None:
# if we just started the video. usually debugging
self.lastMode = "racing"
self.manager.state("mode", "racing")
return
else:
# last mode either racing or setup
# could be either overview or endSequence
isOverview = False
overviewCrop = util.crop(self.window,
np.hstack(self.overviewRect))
gray = cv2.cvtColor(overviewCrop, cv2.COLOR_BGR2GRAY)
t = np.zeros(overviewCrop.shape)
if (np.sum(gray < 2) > 200):
isOverview = True
if isOverview:
self.lastMode = "overview"
self.manager.state("mode", "overview")
else:
self.lastMode = "endSequence"
self.manager.state("mode", "endSequence")
return
def multiperson(img, func, mode):
if mode == 'multi':
scales = [2., 1.8, 1.5, 1.3, 1.2, 1., 0.7, 0.5, 0.25]
else:
scales = [1]
height, width = img.shape[0:2] # 582,800； 这里假设height,width中最大的为800，以下的shape都是依据800计算的
center = (width/2, height/2)
dets, tags = None, []
for idx, i in enumerate(scales):
scale = max(height, width)/200
inp_res = int((i * max(512, max(height, width)) + 3)//4 * 4)
res = (inp_res, inp_res) # 800,800
inp = crop(img, center, scale, res)
tmp = func([inp, inp[:,::-1]])
det = tmp['det'][0,:,:] + tmp['det'][1,:,::-1][:,:,flipRef] #det shape [200,200,17]
if idx == 0:
dets = det
mat = get_transform(center, scale, res)[:2] # shape:[2,3] ;mat=[1 0 0;0 1 109],表示没有缩放，只有y轴方向偏移109=(800-582)/2
mat = np.linalg.pinv(np.array(mat).tolist() + [[0,0,1]])[:2] # 计算仿射变换矩阵mat的伪逆
else:
dets = dets + resize(det, dets.shape[0:2])
if abs(i-1)<0.5:
res = dets.shape[0:2] # res shape:[200,200]
tags += [resize(tmp['tag'][0,:,:,:], res), resize(tmp['tag'][1,:,::-1][:,:,flipRef], res)]
tags = np.concatenate([i[:,:,:,None] for i in tags], axis=3) #tags shape:[200,200,17,2]
dets = dets/len(scales)/2
import cv2
cv2.imwrite('det.jpg', (tags.mean(axis=3).mean(axis=2) *255).astype(np.uint8))
grouped = group(dets, tags, 30)
# grouped shape:[30,17,5] 表示检测到30个人，每个人17个关键点，每个关键点前三个数表示x，y，prediction
grouped[:,:,:2] = kpt_affine(grouped[:,:,:2], mat)
# 筛选并整合人体关键点信息
persons = []
for val in grouped: # val为某一个人的关键点信息
if val[:, 2].max()>0: # 某个人的17个关键点中最大的prediction必须大于0
tmp = {"keypoints": [], "score":float(val[:, 2].mean())} # 将17个关键点的平均值作为score分数值
for j in val: # j表示17个关键点中的某一个
if j[2]>0.: # 关键点的prediction必须大于0，否则认为检测错误，记为[0,0,0]
tmp["keypoints"]+=[float(j[0]), float(j[1]), float(j[2])]
else:
tmp["keypoints"]+=[0, 0, 0]
persons.append(tmp)
return persons # 返回满足要求的所有人
def multiperson(img, func, mode):
if mode == 'multi':
scales = [2., 1.8, 1.5, 1.3, 1.2, 1., 0.7, 0.5, 0.25]
else:
scales = [1]
height, width = img.shape[0:2]
center = (width/2, height/2)
dets, tags = None, []
for idx, i in enumerate(scales):
scale = max(height, width)/200
inp_res = int((i * max(512, max(height, width)) + 3)//4 * 4)
res = (inp_res, inp_res)
inp = crop(img, center, scale, res)
tmp = func([inp, inp[:,::-1]])
det = tmp['det'][0,:,:] + tmp['det'][1,:,::-1][:,:,flipRef]
if idx == 0:
dets = det
mat = get_transform(center, scale, res)[:2]
mat = np.linalg.pinv(np.array(mat).tolist() + [[0,0,1]])[:2]
else:
dets = dets + resize(det, dets.shape[0:2])
if abs(i-1)<0.5:
res = dets.shape[0:2]
tags += [resize(tmp['tag'][0,:,:,:], res), resize(tmp['tag'][1,:,::-1][:,:,flipRef], res)]
tags = np.concatenate([i[:,:,:,None] for i in tags], axis=3)
dets = dets/len(scales)/2
#import cv2
#cv2.imwrite('det.jpg', (tags.mean(axis=3).mean(axis=2) *255).astype(np.uint8))
grouped = group(dets, tags, 30)
grouped[:,:,:2] = kpt_affine(grouped[:,:,:2], mat)
persons = []
for val in grouped:
if val[:, 2].max()>0:
tmp = {"keypoints": [], "score":float(val[:, 2].mean())}
for j in val:
if j[2]>0.:
tmp["keypoints"]+=[float(j[0]), float(j[1]), float(j[2])]
else:
tmp["keypoints"]+=[0, 0, 0]
persons.append(tmp)
return persons
def test(m):
m.eval()
with torch.no_grad():
d = DataLoader(Sun(), 1)
losss = []
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
out = m([y, y, k, s])
out = crop(out, k)
out = center_crop(out, *g.shape[-2:])
loss = npsnr(out, g)
losss.append(-loss.detach().item())
log("psnr", losss[-1])
show(
torch.cat(
(center_crop(y, *g.shape[-2:])[0, 0], g[0, 0], out[0, 0]),
1))
log("psnr avg", sum(losss) / len(losss))
def compress_image(image, window, cutoff, output=OUTPUT_COMPRESSED_IMAGE):
gray = util.im2gray(image)
cropped = util.crop(gray, window)
shaped = util.blockshaped(cropped, window)
dct = fft.dctn(shaped, axes=[1, 2], type=2, norm='ortho')
compressed, mask = util.compress(dct, cutoff)
idct = fft.idctn(compressed, axes=[1, 2], type=2, norm='ortho')
info = np.iinfo(image.dtype)
to_range = (info.min, info.max)
if output == OUTPUT_DCT:
out = dct
from_range = (out.min(), out.max())
elif output == OUTPUT_MASK:
out = mask
from_range = (out.min(), out.max())
elif output == OUTPUT_COMPRESSED_DCT:
out = compressed
from_range = (out.min(), out.max())
elif output == OUTPUT_COMPRESSED_IMAGE:
out = idct
from_range = to_range # No interpolation needed, just clipping
else:
raise ValueError("Invalid value for 'output' parameter.")
# If min == max, min should map to 0
if from_range[0] == from_range[1]:
from_range = (from_range[0], from_range[1] + 1)
normalized = np.interp(out, from_range, to_range)
converted = np.round(normalized).astype(image.dtype)
if output == OUTPUT_MASK:
result = converted
else:
result = util.unblockshaped(converted, cropped.shape)
return result
def encode(self, image):
results = []
boundingboxes = [[]]
if self.detection:
assert self.detect_model is not None, "detect model is not defined"
boundingboxes = self.detect_model.detect(image)
for box in boundingboxes:
if box:
label, score, bbox = box
image_crop = util.crop(image, bbox)
else:
label, score, bbox = None, None, None
image_crop = image
feat = self.extract_model.forward(image_crop)
results.append(dict(label=label,
feat=util.normlize(feat[0]),
boundingbox=bbox
)
)
break
return results
def update(self):
gray = cv2.cvtColor(self.window, cv2.COLOR_BGR2GRAY)
win = gray
vals = []
crop = util.crop(win, np.hstack((self.topLeft[self.direction], size)))
area = cv2.Canny(crop, 350, 300)
luminance = np.mean(crop)
lumWeight = np.clip((1-((luminance-30)/80)), 0, 1)
for index, template in enumerate(templates):
darkWeight = self.darkWeights[index] * lumWeight
val = np.max(scipy.signal.correlate2d(area, template, mode="valid")/float(np.sum(template)))
vals.append(val + (val * darkWeight))
self.currentRank = np.argmax(vals) + 1
certainty = abs(np.subtract(*(np.sort(vals)[-2:])))
self.manager.state('unverifiedRank', self.currentRank, supress=True)
self.manager.state('rankCertainty', certainty, supress=True)
self.lastRank = self.currentRank
def update(self):
gray = cv2.cvtColor(self.window, cv2.COLOR_BGR2GRAY)
win = gray
vals = []
crop = util.crop(win, np.hstack((self.topLeft[self.direction], size)) + (-5, -5, 10, 10))
area = cv2.Canny(crop, 400, 300)
for index, template in enumerate(templates):
val = np.max(scipy.signal.correlate2d(area, template, mode="valid"))/float(np.sum(template))
vals.append(val)
#if self.manager.id == 2:
# print vals
self.lap = np.argmax(vals) + 1
if self.lastLap == self.lap:
self.verified += 1
else:
self.verified = 0
self.lastLap = self.lap
if self.verified > 2:
self.manager.state('lap', self.lap);
def refinement(img, pred_dict, func):
for idx, person in enumerate(pred_dict):
pred = np.array( person['keypoints'] ).reshape(-1, 3)
pts = np.array(pred).reshape(-1, 3)
bbox = pts[pts[:, 2]>0]
x1, x2, y1, y2 = bbox[:, 0].min(), bbox[:, 0].max(), bbox[:, 1].min(), bbox[:, 1].max()
scale = max(max(x2 -x1, y2 -y1), 60) * 1.3/200
center = (x2 + x1)/2, (y1 + y2)/2.
res = (512, 512)
det = []
for res_idx, res in enumerate([(512, 512), (400, 400), (300, 300)]):
inp = crop(img, center, scale, res)
if res_idx == 0:
mat = np.linalg.inv(get_transform(center, scale, res))[:2]
tmp = func([inp, inp[:,::-1]])
det.append( (tmp['det'][0,:,:] + tmp['det'][1,:,::-1][:,:,flipRef])/2 )
det = np.mean([resize(i, det[0].shape[:2]) for i in det], axis = 0)
old = pred.copy()
match_keypoint = []
for i in range(17):
tmp = det[:,:,i]
x, y = np.unravel_index( np.argmax(tmp), tmp.shape )
val = tmp.max()
x, y = kpt_affine( adjustKeypoint(tmp, ([x],[y]))[:,::-1] * 4, mat )[0]
if pred[i, 2] > 0:
match_keypoint.append(np.sum((pred[i, :2] - np.array((x, y)))**2) < 100)
if pred[i, 2] ==0 or val >0.15:
pred[i, :3] = (x, y, val)
if np.mean(match_keypoint) < 0.2:
pred = old
pred_dict[idx]['keypoints'] = pred.reshape(-1).tolist()
return pred_dict
def transform(self, frame):
return util.crop(frame, np.hstack(self.rect))
def _adjoint(self, input):
return util.crop(input, self.ishape, shift=self.shift, center=self.center)
# Проверка, соответствует ли PDF известному формату пластины
##----------------------------------------------------------------
# TODO Проверка, соответствует ли PDF известному формату пластины
# Compress via gs and crop via perl script.
# preview_abs_path has a full path to compressed and cropped pdf
#----------------------------------------------------------------
previewname = pdfName + '.' + outputter_ftp.name + pdfExtension
previewtempname = tempdir + pdfName + '.' + outputter_ftp.name + '.temp' + pdfExtension
preview_abs_path = tempdir + pdfName + '.' + outputter_ftp.name + pdfExtension
crop(pdf_abs_path, previewtempname)
gs_compress = "gs -q -sDEVICE=pdfwrite -dDownsampleColorImages=true " \
"-dColorImageResolution=120 -dCompatibilityLevel=1.4 "\
"-dNOPAUSE -dBATCH -sOutputFile={output} {input}"\
.format(input=previewtempname, output=preview_abs_path)
os.system(gs_compress)
os.unlink(previewtempname)
### CUSTOM OPERATION DEPENDS ON OUTPUTTER
#----------------------------------------
if outputter_ftp.name == 'Leonov':
if machine.name == 'Speedmaster':
outputter_ftp.todir = '_1030x770'
prev_ex = (None,None)
for ex in examples:
pre_out = "/tmp/output/preprocessing/" + str(ex)
car_out = "/tmp/output/infer_car/" + str(ex)
road_out = "/tmp/output/infer_road/" + str(ex)
augment_out = "/tmp/output/augment/" + str(ex)
img = util.read_train_image(ex)
util.write_image(pre_out + ".png", img)
util.write_image(car_out + ".png", img)
util.write_image(road_out + ".png", img)
util.write_image(augment_out + ".png", img)
road_mask, car_mask = util.read_masks(ex)
util.write_mask(car_out + "_truth.png",car_mask)
util.write_mask(road_out + "_truth.png",road_mask)
cropped = util.crop(img,util.preprocess_opts)
util.write_image(pre_out + "_crop.png", cropped)
uncropped = util.uncrop_image(cropped,util.preprocess_opts)
util.write_image(pre_out + "_uncrop.png", uncropped)
preprocessed = util.preprocess_input_image(img,util.preprocess_opts)
car_infer = car_model.predict(np.array([preprocessed]), batch_size=1)[0]
car_infer = util.postprocess_output(car_infer, util.preprocess_opts)
util.write_probability(car_out + "_infer.png", car_infer)
road_infer = road_model.predict(np.array([preprocessed]), batch_size=1)[0]
road_infer = util.postprocess_output(road_infer, util.preprocess_opts)
util.write_probability(road_out + "_infer.png", road_infer)
for aug_name in training.all_augmentations.keys():
write_augmentation(ex, img, car_mask, road_mask, prev_ex, aug_name, training.all_augmentations[aug_name])
for i in range(1,3):
write_augmentation(ex, img, car_mask, road_mask, prev_ex, "pick" + str(i), training.pick_n_augmentations(i))
prev_ex = (img,car_mask)
def process_data_batch(filenames,
text_data,
numeric_data,
desired_shape=(299, 299, 3),
verbose=True,
mode='train'):
i = 0
img_arrays = []
x_shapes = []
y_shapes = []
zpid_list = {}
if verbose:
print('Processing data batch')
count = 0
if verbose:
sys.stdout.write('<')
sys.stdout.flush()
for file in filenames:
if count % 100 == 0:
if verbose:
sys.stdout.write('=')
sys.stdout.flush()
count += 1
try:
folder = 'imgs/'
if mode == 'val':
folder = mode + '_' + folder
if mode == 'test':
folder = mode + '_' + folder
img = Image.open(folder + file)
except OSError:
continue
data = np.array(img)
if data.shape != (300, 400, 3):
# skip if improper shape. most are 300 x 400
continue
zpid = file[:-4]
if zpid not in text_data.keys() or zpid not in numeric_data.keys():
continue
x_shapes.append(data.shape[0])
y_shapes.append(data.shape[1])
assert (zpid not in zpid_list.keys())
zpid_list[zpid] = i
assert (i == len(img_arrays))
img_arrays.append(data)
i += 1
assert (len(img_arrays) == len(zpid_list.keys()))
if verbose:
sys.stdout.write('>\n')
sys.stdout.flush()
N = len(zpid_list)
if verbose:
print('N is {}'.format(N))
ordered_descriptions = [''] * N
ordered_addresses = [''] * N
n_numeric_features = len(next(iter(numeric_data.values())))
ordered_numeric_data = np.zeros((N, n_numeric_features))
for zpid in zpid_list.keys():
index = zpid_list[zpid]
ordered_descriptions[index] = text_data[zpid][0]
ordered_addresses[index] = text_data[zpid][1]
ordered_numeric_data[index] = numeric_data[zpid]
imgs = np.zeros((N, desired_shape[0], desired_shape[1], desired_shape[2]),
dtype=np.uint8)
for i in range(N):
returned_image = util.crop(img_arrays[i],
shape=(desired_shape[0], desired_shape[1]),
random=True)
imgs[i] = returned_image
return imgs, ordered_numeric_data, ordered_descriptions, ordered_addresses
def pre_process(self, image):
if self.detect_model:
bbox = self.detect_model.detect(image)
image = util.crop(image, bbox)
return image
def pre_process(self, image, boundingbox):
if boundingbox:
bbox = list(map(int, boundingbox.split(',')))
image = util.crop(image, bbox)
return image
import numpy as np
import util
from PIL import Image
def partition(x, num):
bins = np.linspace(0, np.max(x), num=num)
print(bins)
y = np.digitize(x, bins, right=True)
return y
if __name__ == "__main__":
#img = np.random.random((10,10,3))
img = Image.open(
'C:/Users/Juan/Documents/17-18 _Stanford/Port 2A/ilha.jpg')
img = np.array(img)
new_img = util.crop(img, (300, 300), True)
print(new_img.shape)
img = Image.fromarray(new_img, 'RGB')
img.show()
print(new_img)
print('\n\n\n')
#print(imgs[i])
#img = Image.fromarray(imgs[i], 'RGB')
#img.show()
exit()
def load_frame(self, seq, frame, mask1_cropped, mask1_crop_param):
cprint('FRAME = ' + str(frame), bcolors.WARNING)
#reading first frame
fresh_mask = True
frame1_dict = read_davis_frame(self.sequences, seq, frame)
image1 = frame1_dict['image']
mask1 = frame1_dict['mask']
if (mask1 > .5).sum() < 500:
return None
if mask1_cropped is not None and mask1_crop_param is not None:
#convert mask1 to its original shape using mask1_crop_param
uncrop_mask1 = crop_undo(mask1_cropped, **mask1_crop_param)
inter = np.logical_and((mask1 > .5), uncrop_mask1 > .5).sum()
union = np.logical_or((mask1 > .5), uncrop_mask1 > .5).sum()
if float(inter) / union > .1:
mask1 = uncrop_mask1
fresh_mask = False
#reading second frame
frame2_dict = read_davis_frame(self.sequences, seq, frame + 1,
self.flow_method)
image2 = frame2_dict['image']
mask2 = frame2_dict['mask']
if not frame2_dict.has_key('iflow'):
frame2_dict['iflow'] = np.zeros(
(image2.shape[0], image2.shape[1], 2))
# Cropping and resizing
mask1[mask1 < .2] = 0
mask1_bbox = bbox(mask1)
cimg = crop(image1,
mask1_bbox,
bbox_enargement_factor=self.bb1_enlargment,
output_shape=self.resizeShape1,
resize_order=3) - self.mean
cmask = crop(mask1.astype('float32'),
mask1_bbox,
bbox_enargement_factor=self.bb1_enlargment,
output_shape=self.resizeShape1)
if self.noisy_mask and fresh_mask:
#print 'Adding Noise to the mask...'
cmask = add_noise_to_mask(cmask)
cimg_masked = cimg * (cmask[:, :, np.newaxis] > self.mask_threshold)
cimg_bg = cimg * (cmask[:, :, np.newaxis] <= self.mask_threshold)
nimg = crop(image2,
mask1_bbox,
bbox_enargement_factor=self.bb2_enlargment,
output_shape=self.resizeShape2,
resize_order=3) - self.mean
label = crop(mask2.astype('float32'),
mask1_bbox,
bbox_enargement_factor=self.bb2_enlargment,
output_shape=self.resizeShape2,
resize_order=0)
label_crop_param = dict(bbox=mask1_bbox,
bbox_enargement_factor=self.bb2_enlargment,
output_shape=image1.shape[0:2])
cmask -= self.mask_mean
if self.bgr:
cimg = cimg[:, :, ::-1]
cimg_masked = cimg_masked[:, :, ::-1]
cimg_bg = cimg_bg[:, :, ::-1]
nimg = nimg[:, :, ::-1]
if self.scale_256:
cimg *= 255
cimg_masked *= 255
cimg_bg *= 255
nimg *= 255
cmask *= 255
item = {
'current_image': cimg.transpose((2, 0, 1)),
'fg_image': cimg_masked.transpose((2, 0, 1)),
'bg_image': cimg_bg.transpose((2, 0, 1)),
'current_mask': cmask,
'next_image': nimg.transpose((2, 0, 1)),
'label': label,
'label_crop_param': label_crop_param
}
#crop inv_flow
if len(self.flow_params) > 0:
inv_flow = frame2_dict['iflow']
max_val = np.abs(inv_flow).max()
if max_val != 0:
inv_flow /= max_val
iflow = crop(inv_flow,
mask1_bbox,
bbox_enargement_factor=self.bb2_enlargment,
resize_order=1,
output_shape=self.resizeShape2,
clip=False,
constant_pad=0)
x_scale = float(iflow.shape[1]) / (mask1_bbox[3] - mask1_bbox[2] +
1) / self.bb2_enlargment
y_scale = float(iflow.shape[0]) / (mask1_bbox[1] - mask1_bbox[0] +
1) / self.bb2_enlargment
for i in range(len(self.flow_params)):
name, down_scale, offset, flow_scale = self.flow_params[i]
pad = int(-offset + (down_scale - 1) / 2)
h = np.floor(float(iflow.shape[0] + 2 * pad) / down_scale)
w = np.floor(float(iflow.shape[1] + 2 * pad) / down_scale)
n_flow = np.pad(
iflow, ((pad, int(h * down_scale - iflow.shape[0] - pad)),
(pad, int(h * down_scale - iflow.shape[1] - pad)),
(0, 0)), 'constant')
n_flow = resize(n_flow, (h, w),
order=1,
mode='nearest',
clip=False)
n_flow[:, :, 0] *= max_val * flow_scale * x_scale / down_scale
n_flow[:, :, 1] *= max_val * flow_scale * y_scale / down_scale
n_flow = n_flow.transpose((2, 0, 1))[::-1, :, :]
item[name] = n_flow
return item
from util import load_face_images, crop
from display_outputs import display_outputs
from photometric_stereo import *
output_dir = './outputs/'
num_image = 11
# load data
amb_img, img_arr, light_dirs = load_face_images('./data/', 'yaleB01', num_image)
# preprocess the data:
# subtract ambient_image from each image in imarray
img_arr = img_arr - amb_img
# make sure no pixel is less than zero
img_arr[img_arr < 0] = 0
# rescale values in imarray to be between 0 and 1
img_arr /= 255
# Crop the image so that only fae regions remain while the background and hair regions are excluded
img_arr = crop(img_arr, 255, 120, 180, 170)
# get albedo and normal
albedo_img, normal_map = photometric_stereo(img_arr, light_dirs)
(albedo_img_un, normal_map_un), light_dirs_un = uncalibrated_photometric_stereo(img_arr)
display_outputs(output_dir, num_image, img_arr, albedo_img, normal_map, light_dirs, albedo_img_un, normal_map_un, light_dirs_un)
def load_next_image(self):
if self.cur == self.db['length']:
self.cur = 0
if self.shuffle:
random.shuffle(self.indexes)
img_cur, m = self.instance_loader(self.db, self.cur)
rmin, rmax, cmin, cmax = bbox(m)
base_tran = self.video_transformer.sample()
frame1_tran = base_tran + self.frame_transformer.sample(x_scale = cmax+1-cmin, y_scale = rmax+1-rmin)
frame2_tran = base_tran + self.frame_transformer.sample(x_scale = cmax+1-cmin, y_scale = rmax+1-rmin)
image1 = frame1_tran.transform_img(img_cur.copy(), img_cur.shape[:2], m)
mask1 = frame1_tran.transform_mask(m.copy(), m.shape)[0]
mask1[mask1 == -1] = 0
image2 = frame2_tran.transform_img(img_cur.copy(),img_cur.shape[:2], m)
mask2 = frame2_tran.transform_mask(m.copy(), m.shape)[0]
mask2[mask2 == -1] = 0
if mask1.sum() < 1000:
self.cur += 1
return self.load_next_image()
# Cropping and resizing
mask1_bbox = bbox(mask1)
cimg = crop(image1, mask1_bbox, bbox_enargement_factor = self.img1_bb_enlargement, output_shape = self.resizeShape1, resize_order = 3) - self.mean
cmask = crop(mask1.astype('float32'), mask1_bbox, bbox_enargement_factor = self.img1_bb_enlargement, output_shape = self.resizeShape1, resize_order = 0)
if self.noisy_mask:
cmask = add_noise_to_mask(cmask)
cimg_masked = (cimg * cmask[:,:,np.newaxis])
nimg = crop(image2, mask1_bbox, bbox_enargement_factor = self.img2_bb_enlargement, output_shape = self.resizeShape2, resize_order = 3) - self.mean
label = crop(mask2.astype('float32'), mask1_bbox, bbox_enargement_factor = self.img2_bb_enlargement, output_shape = self.resizeShape2, resize_order = 0)
#We are interested in inverse flow
#Which are frame2-->frame1 flow fields
# I1 = T1(I)
# I2 = T2(I)
# p1 = T1(T2^-1(p2))
# flow(p2) = p1 - p2
iflow = None
if self.inverse_flow:
newx = np.arange(image2.shape[1])
newy = np.arange(image2.shape[0])
mesh_grid = np.stack(np.meshgrid(newx, newy), axis = 0)
locs2 = mesh_grid
x,y = frame2_tran.itransform_points(locs2[0].ravel(), locs2[1].ravel(), locs2[0].shape)
x,y = frame1_tran.transform_points(x, y, locs2[0].shape)
locs1 = np.concatenate((x,y)).reshape((2,) + locs2[0].shape)
flow = locs1 - locs2
iflow = crop(flow.transpose((1,2,0)), mask1_bbox, bbox_enargement_factor = self.img2_bb_enlargement, resize_order=1, output_shape = self.resizeShape2, clip = False, constant_pad = 0)
iflow[:, :, 0] *= float(iflow.shape[1]) * self.flow_scale /(mask1_bbox[3] - mask1_bbox[2] + 1)/self.img2_bb_enlargement
iflow[:, :, 1] *= float(iflow.shape[0]) * self.flow_scale /(mask1_bbox[1] - mask1_bbox[0] + 1)/self.img2_bb_enlargement
#print ' flow', flow[0].max(), flow[0].min(), flow[1].max(), flow[1].min()
#print 'iflow', iflow[:,:,0].max(), iflow[:,:,0].min(), iflow[:,:,1].max(), iflow[:,:,1].min()
#from util import write_flo_file
#from skimage.io import imsave
#imsave('im0.png', cimg)
#imsave('im1.png', nimg)
#write_flo_file('inv0.flo', iflow)
#Apply mask
iflow = iflow.transpose((2,0,1))[::-1, :, :]
#iflow[0][label == 0] = np.nan
#iflow[1][label == 0] = np.nan
self.cur += 1
if self.bgr:
cimg = cimg[:,:, ::-1]
cimg_masked = cimg_masked[:, :, ::-1]
nimg = nimg[:, :, ::-1]
if self.scale_256:
cimg *= 255
cimg_masked *= 255
nimg *= 255
item = {'current_image':cimg.transpose((2,0,1)),
'current_mask' :cimg_masked.transpose((2,0,1)),
'next_image' :nimg.transpose((2,0,1)),
'mask' :cmask[np.newaxis, :, :],
'label' :label[np.newaxis, :, :],
'inverse_flow' :iflow}
return item