def construct_network():
## Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, HEIGHT, WIDTH, CHANNELS))
## Construct the network
net = models.ResNet50UpProj({'data': input_node}, BATCH_SIZE, 1, False)
return net, input_node
def predict(model_data_path, image_cv2, channels=3, batch_size=1):
dims = image_cv2.shape
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32, shape=(None, dims[1], dims[2], channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: image_cv2})
return pred
def start_network(model_data_path):
"""
Isolated function to start the network and return it as a stucture.
This is to make it re-runnable on multiple images using predict_from_image.
"""
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Create a placeholder for the input image
input_node = tf.compat.v1.placeholder(dtype=tf.float32,
shape=(1, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node},
batch_size,
1,
False,
trainable=False)
sess = tf.compat.v1.Session()
# Load the converted parameters
print('Loading the model')
net.load(model_data_path, sess)
#using a namedtuple to return the network state.
Network = collections.namedtuple('Network', ['sess', 'net', 'input_node'])
network = Network(sess, net, input_node)
print('returning the model')
return network
def predict(model_data_path, image_path, gt_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Read image
gt = Image.open(gt_path)
gt = gt.resize([160, 128], Image.ANTIALIAS)
gt = np.array(gt).astype('float32')
min_max_scaler = scale.MinMaxScaler()
gt = min_max_scaler.fit_transform(gt)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
print(pred.shape)
a = pred[0, :, :, 0]
a = util.invert(a)
a = min_max_scaler.fit_transform(a)
print('RMS error.. the best value is 0.0')
rms = sqrt(mean_squared_error(gt, a))
print(rms)
# Plot result
fig = plt.figure()
ii = plt.imshow(a, cmap='gray', interpolation='nearest')
fig.colorbar(ii)
plt.show()
return pred
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
cap = cv2.imread(image_path)
orig_height = cap.shape[0]
orig_width = cap.shape[1]
background = Image.new('RGB', (width, height), (0, 0, 0))
if (orig_width / orig_height < width / height):
scale = height * 1.0 / orig_height
new_width = int(orig_width * scale)
offset = (int(round(((width - new_width) / 2), 0)), 0)
img = img.resize([new_width, height], Image.ANTIALIAS)
else:
scale = width * 1.0 / orig_width
new_height = int(orig_height * scale)
offset = (0, int(round(((height - new_height) / 2), 0)))
img = img.resize([width, new_height], Image.ANTIALIAS)
background.paste(img, offset)
img = np.array(background).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# Plot result
fig = plt.figure()
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
plt.show()
return pred
def export(model_data_path, export_dir):
height = 228
width = 304
channels = 3
batch_size = 1
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
net = models.ResNet50UpProj({'data': input_node}, batch_size)
builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
net.load(model_data_path, sess)
uninitialized_vars = []
for var in tf.global_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninitialized_vars.append(var)
init_new_vars_op = tf.variables_initializer(uninitialized_vars)
sess.run(init_new_vars_op)
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING])
print('Saving the graph')
builder.save()
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image(OpenCV)
img = cv2.imread(image_path)
img = cv2.resize(img, (width, height))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img.astype('float32')
img = np.expand_dims(img, axis=0)
# Read image (pillow)
# img = Image.open(image_path)
# img = img.resize([width,height], Image.ANTIALIAS)
# img = np.array(img).astype('float32')
# img = np.expand_dims(np.asarray(img), axis = 0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# Plot result (matplotlib)
# fig = plt.figure()
# ii = plt.imshow(pred[0,:,:,0], interpolation='nearest')
# fig.colorbar(ii)
# plt.show()
# show result (OpenCV)
output = pred[0, :, :, 0]
maxnum = output.max()
output = output / maxnum * 256
output = output.astype('uint8')
cv2.imshow('result', output)
cv2.waitKey(0)
cv2.destroyAllWindows()
return pred
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
# img = Image.open(image_path)
# img = img.resize([width,height], Image.ANTIALIAS)
# img = np.array(img).astype('float32')
# print(img.shape)
# img = np.expand_dims(np.asarray(img), axis = 0)
# rewrite image read code
img = misc.imread(image_path)
img = misc.imresize(img, [height, width], interp='nearest')
origin = img
img = np.array(img).astype('float32')
img = img[:, :, 1:4]
img = np.expand_dims(np.asarray(img), axis=0)
print(img.shape)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
print('Model loaded')
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# Plot result
fig = plt.figure()
origin = plt.imshow(origin, interpolation='nearest')
plt.savefig('test_origin.png')
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
plt.savefig('test_result.png')
# plt.show()
return pred
def predict(model_data_path, image_folder):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
for root, dirs, files in os.walk(image_folder):
for file in files:
directory = root + '/'
if 'jpg' in file and file[0] is not '.':
image_path = directory + file
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(),
feed_dict={input_node: img})
pred_depth = pred[0, :, :, 0]
output_name = os.path.splitext(
os.path.basename(image_path))[0]
output_directory = os.path.dirname(image_path)
plt.imsave(os.path.join(
output_directory, "{}_depth.png".format(output_name)),
pred_depth,
cmap='plasma')
np.save(
os.path.join(output_directory,
"{}_depth.npy".format(output_name)),
pred_depth)
print('finished exporting {}'.format(
os.path.join(output_directory, output_name)))
return pred
def predict(model_data_path, image_root, output_root):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
fileNum = len([name for name in os.listdir(image_root)])
names = [name for name in os.listdir(image_root)]
print(image_root)
print(fileNum)
#for idx in range(0, fileNum):
for nm in names:
#image_path = image_root + "color_out" + str(idx+1).zfill(6) + ".png"
image_path = image_root + nm
#output_path = output_root + "depth_out" + str(idx+1).zfill(6) + ".png"
output_path = output_root + nm
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
pre_new = pred[0, :, :, 0]
pre_new = pre_new * 1000 * 5
pre_resize = scipy.ndimage.zoom(pre_new, 4, order=0)
img = np.zeros((128 * 4, 160 * 4, 1), dtype=np.uint16)
img[:, :, 0] = pre_resize
numpngw.write_png(output_path, img)
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img.save("/tmp/input.png", "PNG")
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
net.load(model_data_path, sess)
uninitialized_vars = []
for var in tf.global_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninitialized_vars.append(var)
init_new_vars_op = tf.variables_initializer(uninitialized_vars)
sess.run(init_new_vars_op)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# Write the result to file
output = pred[0, :, :, 0].astype(float)
output *= (255. / output.max())
output = output.astype('uint8')
outImg = Image.fromarray(output, 'L')
outImg.save("/tmp/output.png")
# Plot result
fig = plt.figure()
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
plt.show()
return pred
def predict(model_data_path, image_path, orig):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
fig2 = plt.figure(2)
img2 = Image.open(orig)
img2 = img2.resize([width, height], Image.NEAREST)
img2 = np.array(img2).astype('float32') / 1000
ii = plt.imshow(img2, interpolation='nearest')
fig2.colorbar(ii)
plt.show(block=False)
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
net.load(model_data_path, sess)
uninitialized_vars = []
for var in tf.global_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninitialized_vars.append(var)
init_new_vars_op = tf.variables_initializer(uninitialized_vars)
sess.run(init_new_vars_op)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
print("plotting results")
# Plot result
fig = plt.figure(1)
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
plt.show()
return pred
def predict(model_data_path, image_path):
"""
The main predict function present in the original source file.
It either opens a file or interprets a given image for its input.
Then it sets up the neural network, runs a prediction, and plots its output.
"""
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# read cv image
# img = Image.fromarray(image_path)
# img = img.resize([width,height], Image.ANTIALIAS)
# img = np.array(img).astype('float32')
# img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.compat.v1.placeholder(dtype=tf.float32,
shape=(1, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
sess = tf.compat.v1.Session()
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
# saver = tf.compat.v1.train.Saver()
# saver.restore(sess, model_data_path)
# Use to load from npy file
net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img[:, :, :]})
# Plot result
fig = plt.figure()
plt_im = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(plt_im)
plt.show()
return pred
def main():
# Read image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
# saver = tf.train.Saver()
# saver.restore(sess, model_data_path)
plt.ion()
# Use to load from npy file
net.load(
"D:\\ME5001_project\\FCRN-DepthPrediction-master\\image_depth_prediction\\NYU_ResNet-UpProj.npy",
sess)
while (True):
ret, img = cap.read()
cv2.imshow('frame1', img)
# print('Original Dimensions : ', img.shape) resize image
resized = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
img = np.expand_dims(np.asarray(resized), axis=0)
# Create a placeholder for the input imageqqqqqqQq
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
print('Plot')
# Plot result
fig = plt.figure()
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
fig.canvas.draw()
data = np.fromstring(fig.canvas.tostring_rgb(),
dtype=np.uint8,
sep='')
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
#final_img = Image.fromarray(data, 'RGB')
plt.close()
#gray = cv2.cvtColor(data, cv2.COLOR_BGR2GRAY)
#ret,imggray = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY)
cv2.imshow('frame', data)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
os._exit(0)
def predict_multiple(model_data_path, img):
# Default input size
#height = 228
#width = 304
height = 224
width = 224
channels = 3
#batch_size = 1
batch_size = img.shape[0]
# Read image
#img = Image.open(image_path)
#for image in img:
#print(image.shape)
#image = image.resize([width,height], Image.ANTIALIAS)
#image = np.array(image).astype('float32')
#img = np.array(img)
#print(img.shape)
#img = np.expand_dims(np.asarray(img), axis = 0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
# Load the converted parameters
print('Loading the depth prediction model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
#print(pred.shape)
return pred
def __init__(self,
path_model=FCRN_MODEL_PATH,
image_shape=FCRN_IMAGE_SHAPE,
channels=3,
batch_size=1,
verbose=False):
'''Construct TensorFlow graph to predict depth in the future.
Args:
path_model (str):
path to pretrained model weights
path_input (str):
path to the directory with input raw RGB images
path_output (str):
path to the directory the depth dataset will be stored into
channels (int):
number of image channels, such as RGB (channels == 3)
batch_size (int):
size of the batch size for ResNet
verbose (bool):
toggle print debug information to the console
'''
self.verbose = verbose
self.path_model = path_model
# Remember tensorflow session
self.batch_size = batch_size
self.channels = channels
self.saver = None
# Create a placeholder for the input image
self.input_node = tf.placeholder(tf.float32,
shape=(None, image_shape[0],
image_shape[1], channels))
if self.verbose:
print('[FCRN]: Input tensorflow placeholder created...')
# Construct the network
self.net = models.ResNet50UpProj({'data': self.input_node},
self.batch_size, 1, False)
if self.verbose:
print('[FCRN]: ResNet50 initialized...')
def predict_camera(model_data_path):
# default input size
height = 480
width = 640
channels = 3
batch_size = 1
# create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
# get video stream
cap = cv2.VideoCapture(1)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Restore the training parameters from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
while True:
ret, frame = cap.read()
img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img = img.astype('float32')
img = np.expand_dims(img, axis=0)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
dst = pred[0, :, :, 0]
maxnum = dst.max()
dst = dst / maxnum * 256
dst = dst.astype('uint8')
cv2.imshow('result', dst)
if (cv2.waitKey(1) & 0xFF == ord('q')):
break
cap.release()
cv2.destroyAllWindows()
def main(argv):
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(arg.batch_size, arg.img_height,
arg.img_width, 3),
name='input_image')
# Load model and get output
model = models.ResNet50UpProj({'data': input_node}, 1, 1, False)
y = model.get_output()
# initialise the saver
saver = tf.train.Saver()
# Session
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
# Initialize variables
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# restore all variables from checkpoint
saver.restore(sess, arg.ckpt_file)
# node that are required output nodes
output_node_names = [arg.output_node_name]
# We use a built-in TF helper to export variables to constants
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess,
tf.get_default_graph().as_graph_def(),
# The graph_def is used to retrieve the nodes
output_node_names # The output node names are used to select the useful nodes
)
# Convert variables to constants
output_graph_def = tf.graph_util.remove_training_nodes(output_graph_def)
# Finally we serialize and dump the output graph to the filesystem
if not os.path.exists(arg.output_dir):
os.makedirs(arg.output_dir)
out_pb_file = os.path.join(arg.output_dir,
Path(arg.ckpt_file).stem + ".pb")
with tf.gfile.GFile(out_pb_file, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("Frozen graph file {} created successfully".format(out_pb_file))
def predict(model_data_path, frame):
# Default input size
height = 480
width = 640
channels = 3
batch_size = 1
# Read image
# img = Image.open(image_path)
img = frame
#img = img.resize([width,height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis = 0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32, shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
tf.get_variable_scope().reuse_variables()
# Plot result
#fig = plt.figure()
#ii = plt.imshow(pred[0,:,:,0], interpolation='nearest')
#fig.colorbar(ii)
#plt.show()
#plt.pause(0.001)
return pred[0,:,:,0]
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# Plot result
fig = plt.figure()
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
plt.savefig('final.jpg')
image = cv2.imread('final.jpg')
graysc = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
cv2.imwrite('finalgray.jpg', graysc)
return pred
def create1():
model_data_path = "/home/kheteshr/Desktop/deep_sort_yolov3-master/NYU_ResNet-UpProj.npy"
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
'''\img = Image.open(image_path)
img = img.resize([width,height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis = 0)
'''
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
#saver = tf.train.Saver()
#saver.restore(sess, model_data_path)
# Use to load from npy file
net.load(model_data_path, sess)
# Evalute the network for the given image
print('in func1')
# Plot result
'''fig = plt.figure()
ii = plt.imshow(pred[0,:,:,0], interpolation='nearest')
fig.colorbar(ii)
plt.show()'''
sess.close()
return net, input_node
def evaluate(model_data_path, image_path):
# Default input size
height = 240
width = 320
channels = 3
output_height, output_width = height, width
for i in range(5):
output_height = np.ceil(output_height / 2)
output_width = np.ceil(output_width / 2)
output_height = int(16 * output_height)
output_width = int(16 * output_width)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, 1, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
x, y = load_data(image_path, (height, width, channels),
(output_height, output_width, 1))
ypred = np.zeros(y.shape)
rel = 0
total_batch = x.shape[0]
for i in range(total_batch):
pred = sess.run(net.get_output(),
feed_dict={input_node: x[i:(i + 1)]})
ypred[i] = pred
# Evalute the network
rel = abs(y - ypred) / y
rel = np.mean(rel)
thresh = np.maximum((y / ypred), (ypred / y))
acc = (thresh < 1.25).mean()
return rel, acc
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# Plot result
formatted = ((pred[0, :, :, 0]) * 255 /
np.max(pred[0, :, :, 0])).astype('uint8')
img = Image.fromarray(formatted)
img.save('my.png')
return pred
def predict(model_data_path, image_path):
print image_path
# Default input size
height = 256
width = 256
channels = 3
batch_size = 1
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
for image in image_path:
# Read image
img = Image.open(image)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
pred = sess.run(net.get_output(), feed_dict={input_node: img})
imsave("../../pix2pix-tensorflow/data/depth/" + image[37:],
pred[0, :, :, 0])
print str(image[37:]) + " saved.."
def predictDepth(image_list, pwd):
depth = []
# Default input size
height = 128 * 2
width = 416 * 2
channels = 3
batch_size = 1
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, pwd + '/laina/NYU_FCRN.ckpt')
# Use to load from npy file
#net.load(model_data_path, sess)
for image in image_list:
# Read image
img = Image.open(image)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
pred = sess.run(net.get_output(), feed_dict={input_node: img})
depth.append(pred[0, :, :, 0])
depth = np.array(depth)
np.save(pwd + "/laina/output/depth.npy", depth)
return depth
def predict(model_data_path, image_path):
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
print('Loading model...')
net.load(model_data_path, sess)
testfileloc = list(glob.glob(image_path + '\input\*'))
for i in testfileloc:
print('predicting...', i)
img = Image.open(i)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
pred = sess.run(net.get_output(), feed_dict={input_node: img})
fig = plt.figure()
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(ii)
# plt.show()
print('saved..', i.replace('input', 'output'))
plt.savefig(i.replace('input', 'output'))
return pred
def predictor():
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
img = Image.open(image_path)
actualwidth, actualheight = img.size
img = img.resize([width, height], Image.ANTIALIAS)
imgsaved = img.resize([160, 128], Image.ANTIALIAS)
imgsaved = np.array(imgsaved).astype('float32')
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
#mycode
outimage = pred[0, :, :, 0]
xmax, xmin = outimage.max(), outimage.min()
outnormimage = (outimage - xmin) / (xmax - xmin)
outnorminvimage = np.ones((np.shape(outnormimage))) - outnormimage
#outnorminvimage = (outnorminvimage - 0.7)
#outnorminvimage[outnorminvimage<0] = 0
#outnorminvimage = outnorminvimage * 3
outnorminvimage = np.power(outnorminvimage, 2)
print(outnorminvimage[60, 60])
outnorminvimage = outnorminvimage[:, :, np.newaxis]
imgsaved1 = np.multiply(imgsaved[:, :, :], outnorminvimage[:, :, :])
#Image.fromarray(np.uint8(imgsaved1)).show()
#fig = plt.figure()
#jj=plt.imshow(outnorminvimage[:,:,0])
#fig.colorbar(jj)
#plt.show()
print(np.shape(outnorminvimage))
fig = plt.figure("basic image into filter in low res")
kk = plt.imshow(Image.fromarray(np.uint8(imgsaved1)))
#extras
reimagefilter = Image.fromarray(
np.uint8(outnorminvimage[:, :, 0] * 255))
resizedfilter = reimagefilter.resize([actualwidth, actualheight],
Image.ANTIALIAS)
resizedfilter = np.array(resizedfilter).astype('float32')
resizedfilter = resizedfilter / 255
resizedfilter = resizedfilter[:, :, np.newaxis]
for x in range(3, actualheight - 3):
for y in range(3, actualwidth - 3):
resizedfilter[x, y, 0] = np.mean(resizedfilter[x - 2:x + 2,
y - 2:y + 2, 0])
#mutiplied darkening
img2 = Image.open(image_path)
img2 = img2.resize([actualwidth, actualheight], Image.ANTIALIAS)
img2 = np.array(img2).astype('float32')
imgsaved2 = np.multiply(img2[:, :, :], resizedfilter[:, :, :])
#Image.fromarray(np.uint8(imgsaved2)).show()
fig = plt.figure("darken filter into image in high res")
kkiss = plt.imshow(Image.fromarray(np.uint8(imgsaved2)))
#hard darkening
bbc = img2.copy()
abc = img2.copy()
for x in range(10, actualheight - 11):
for y in range(10, actualwidth - 10):
g = 0
if resizedfilter[x, y, 0] < 0.3:
g = 5
elif resizedfilter[x, y, 0] < 0.5:
g = 3
if g > 0:
bbc[x, y, 0] = abc[x, y, 0] - g
bbc[x, y, 1] = abc[x, y, 1] - g
bbc[x, y, 2] = abc[x, y, 2] - g
if g == 0:
bbc[x, y, 0] = abc[x, y, 0] + 5
bbc[x, y, 1] = abc[x, y, 1] + 5
bbc[x, y, 2] = abc[x, y, 2] + 5
fig = plt.figure("hard darken image high res")
kbkiss = plt.imshow(Image.fromarray(np.uint8(bbc)))
#extra2
img3 = Image.open(image_path)
img3 = img3.resize([actualwidth, actualheight], Image.ANTIALIAS)
img3 = np.array(img3).astype('float32')
bb = img3.copy()
print(resizedfilter[10, 10, 0])
for x in range(10, actualheight - 11):
for y in range(10, actualwidth - 10):
f = 0
if resizedfilter[x, y, 0] < 0.3:
f = 3
elif resizedfilter[x, y, 0] < 0.5:
f = 2
if f > 0:
bb[x, y, 0] = np.mean(img3[x - f:x + f, y - f:y + f, 0])
bb[x, y, 1] = np.mean(img3[x - f:x + f, y - f:y + f, 1])
bb[x, y, 2] = np.mean(img3[x - f:x + f, y - f:y + f, 2])
#bb=np.multiply(bb[:,:,:],resizedfilter[:,:,:])
#Image.fromarray(np.uint8(bb)).show()
fig = plt.figure("blurred image high res")
bkkiss = plt.imshow(Image.fromarray(np.uint8(bb)))
fig = plt.figure("actual image high res")
bkkiss = plt.imshow(Image.fromarray(np.uint8(img3)))
#dark and blur both
cc = imgsaved2.copy() ###take the already darkened photo
dd = imgsaved2.copy()
print(resizedfilter[10, 10, 0])
#for x in range(10,actualheight-11):
#for y in range(10,actualwidth-10):
#resizedfilter[x,y,0]=np.mean(resizedfilter[x-1:x+1,y-1:y+1,0])
for x in range(10, actualheight - 11):
for y in range(10, actualwidth - 10):
f = 0
if resizedfilter[x, y, 0] < 0.3:
f = 2
elif resizedfilter[x, y, 0] < 0.5:
f = 1
if f > 0:
dd[x, y, 0] = np.mean(cc[x - f:x + f, y - f:y + f, 0])
dd[x, y, 1] = np.mean(cc[x - f:x + f, y - f:y + f, 1])
dd[x, y, 2] = np.mean(cc[x - f:x + f, y - f:y + f, 2])
if f == 0:
dd[x, y, 0] = cc[x, y, 0] + 5
dd[x, y, 1] = cc[x, y, 1] + 5
dd[x, y, 2] = cc[x, y, 2] + 5
#bb=np.multiply(bb[:,:,:],resizedfilter[:,:,:])
#Image.fromarray(np.uint8(bb)).show()
fig = plt.figure("blurred and dark image high res")
pussy = plt.imshow(Image.fromarray(np.uint8(dd)))
#fig = plt.figure("actual image high res")
#bkkiss=plt.imshow(Image.fromarray(np.uint8(img3)))
# Plot result
fig = plt.figure("actual depth estimate")
iih = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
fig.colorbar(iih)
plt.show()
def predict():
model_data_path = 'NYU_FCRN.ckpt'
fileAirPath = './image/'
airPic = os.listdir(fileAirPath)
airPic.sort()
fileRoadPath = './pic/'
roadPic = os.listdir(fileRoadPath)
roadPic = [int(fi.split('.')[0][14:]) for fi in roadPic]
roadPic.sort()
# Default input size
height = 228
width = 304
channels = 3
batch_size = 1
# Read image
imgAll = []
imgSumAll = []
for file in airPic:
img = Image.open(fileAirPath + file)
# for file in roadPic:
# img = Image.open(fileRoadPath + "original_frame" + str(file) + '.bmp')
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
imgAll.append(img)
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
with tf.Session() as sess:
# Load the converted parameters
print('Loading the model')
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_data_path)
# Use to load from npy file
#net.load(model_data_path, sess)
# Evalute the network for the given image
for img in imgAll:
pred = sess.run(net.get_output(), feed_dict={input_node: img})
predImg = pred[0, :, :, 0]
imgSum = predImg.sum()
imgSumAll.append((imgSum / 100))
# break
temp = []
for i in range(24):
for j in range(60):
if j == 0:
date = str(i) + ":0" + str(j)
else:
# date = str(i) + ":" + str(j)
date = ""
temp.append(date)
datetime = temp[:480]
len(datetime)
plt.figure(figsize=(15, 7), dpi=150)
plt.xticks(np.arange(len(datetime)), datetime, rotation=45)
plt.plot(imgSumAll, label="Image depth")
plt.xlabel("Time", fontsize=25)
plt.ylabel("Image depth", fontsize=25)
plt.tick_params(labelsize=15)
plt.legend(fontsize=20)
plt.grid(which='major', axis='y')
plt.savefig('./airDepth.jpg', bbox_inches='tight')
plt.show()
return output_dict
#i=1
#for image_path in TEST_IMAGE_PATHS:
height = 228
width = 304
channels = 3
batch_size = 1
#create a placeholder for the jnput jmage
jnput_node = tf.placeholder(tf.float32, shape=(None, height, width, channels))
#Construct the network
net = models.ResNet50UpProj({'data': jnput_node}, batch_size, 1, False)
print('Loading the m-m-m-model')
sess = tf.Session()
#use to load from the ckpt file
saver = tf.train.Saver()
saver.restore(sess, "NYU_FCRN.ckpt")
#Use to load from npy file
#net.load(model_data_path, sess)
#Evaluate the network for the given image
while (True):
data = urllib.request.urlretrieve("http://localhost:3000/camera_py",
"cam.png")
def predict_video(model_path, video_path, baseline_path, interval,
smoothed_2d):
logger.info("深度推定出力開始")
# 深度用サブディレクトリ
subdir = '{0}/depth'.format(baseline_path)
if os.path.exists(subdir):
# 既にディレクトリがある場合、一旦削除
shutil.rmtree(subdir)
os.makedirs(subdir)
#関節位置情報ファイル
depthf = open(baseline_path + '/depth.txt', 'w')
# Default input size
height = 288
width = 512
channels = 3
batch_size = 1
scale = 0
# # tensorflowをリセットする
# tf.reset_default_graph()
# 映像サイズを取得する
n = 0
cap = cv2.VideoCapture(video_path)
while (cap.isOpened()):
orig_width = cap.get(3) # float
orig_height = cap.get(4) # float
logger.debug("width: {0}, height: {1}".format(orig_width, orig_height))
'''
img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
background = Image.new('RGB', (width, height), (0, 0, 0))
if(orig_width/orig_height<width/height):
scale = height * 1.0 / orig_height
logger.debug("scale: {0}".format(scale))
new_width = int(orig_width * scale)
offset = (int(round(((width - new_width)/2),0)),0)
img = img.resize([new_width, height], Image.ANTIALIAS)
else:
scale = width * 1.0 / orig_width
logger.debug("scale: {0}".format(scale))
new_height = int(orig_height * scale)
offset = (0,int(round(((height - new_height)/2),0)))
img = img.resize([width, new_height], Image.ANTIALIAS)
background.paste(img, offset)
'''
# 縮小倍率
scale = width / orig_width
# logger.debug("scale: {0}".format(scale))
# height = int(orig_height * scale)
# logger.debug("width: {0}, height: {1}".format(width, height))
break
# 再設定したサイズでtensorflow準備
# Create a placeholder for the input image
input_node = tf.placeholder(tf.float32,
shape=(None, height, width, channels))
# Construct the network
net = models.ResNet50UpProj({'data': input_node}, batch_size, 1, False)
png_lib = []
with tf.Session() as sess:
# Use to load from ckpt file
saver = tf.train.Saver()
saver.restore(sess, model_path)
# 動画を1枚ずつ画像に変換する
n = 0
cap = cv2.VideoCapture(video_path)
while (cap.isOpened()):
# 動画から1枚キャプチャして読み込む
flag, frame = cap.read() # Capture frame-by-frame
# キャプチャが終わっていたら終了
if flag == False: # Is a frame left?
break
txt_length = len(smoothed_2d)
if (n == txt_length):
break
if n % interval == 0:
# 先に間引き分同じのを追加
if interval > 1 and n > 0:
for m in range(interval - 1):
# logger.debug("間引き分追加 {0}".format(m))
png_lib.append(
imageio.imread("{0}/depth_{1:012d}.png".format(
subdir, n - interval)))
# 一定間隔フレームおきにキャプチャした画像を深度推定する
logger.info("深度推定: n={0}".format(n))
# キャプチャ画像を読み込む
img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
background = Image.new('RGB', (width, height), (255, 255, 255))
if (orig_width / orig_height < width / height):
scale = height * 1.0 / orig_height
logger.debug("scale: {0}".format(scale))
new_width = int(orig_width * scale)
offset = (int(round(((width - new_width) / 2), 0)), 0)
img = img.resize([new_width, height], Image.ANTIALIAS)
else:
scale = width * 1.0 / orig_width
logger.debug("scale: {0}".format(scale))
new_height = int(orig_height * scale)
offset = (0, int(round(((height - new_height) / 2), 0)))
img = img.resize([width, new_height], Image.ANTIALIAS)
background.paste(img, offset)
img = img.resize([width, height], Image.ANTIALIAS)
img = np.array(img).astype('float32')
img = np.expand_dims(np.asarray(img), axis=0)
# Use to load from npy file
#net.load(model_path, sess)
# Evalute the network for the given image
pred = sess.run(net.get_output(), feed_dict={input_node: img})
# 深度解析後の画像サイズ
pred_height = len(pred[0])
pred_width = len(pred[0][0])
logger.debug("smoothed_2d[n] {0}".format(smoothed_2d[n]))
# 両足の付け根の中間を取得する
smoothed_center_x = np.average(
[smoothed_2d[n][0][0], smoothed_2d[n][1][0]])
smoothed_center_y = np.average(
[smoothed_2d[n][0][1], smoothed_2d[n][1][1]])
logger.debug(
"smoothed_center_x: {0}, smoothed_center_y: {1}".format(
smoothed_center_x, smoothed_center_y))
# オリジナルの画像サイズから、縮尺を取得
scale_orig_x = smoothed_center_x / orig_width
scale_orig_y = smoothed_center_y / orig_height
logger.debug("scale_orig_x: {0}, scale_orig_y: {1}".format(
scale_orig_x, scale_orig_y))
# 縮尺を展開して、深度解析後の画像サイズに合わせる
pred_x = int(pred_width * scale_orig_x)
pred_y = int(pred_height * scale_orig_y)
logger.debug("pred_x: {0}, pred_y: {1}, depth: {2}".format(
pred_x, pred_y, pred[0][pred_y][pred_x][0]))
# 深度ファイルに出力
depthf.write("{0}, {1}\n".format(n,
pred[0][pred_y][pred_x][0]))
# Plot result
plt.cla()
plt.clf()
ii = plt.imshow(pred[0, :, :, 0], interpolation='nearest')
plt.colorbar(ii)
# 散布図のようにして、出力に使ったポイントを明示
plt.scatter(pred_x, pred_y, s=5, c="#FFFFFF")
# 深度画像保存
plotName = "{0}/depth_{1:012d}.png".format(subdir, n)
plt.savefig(plotName)
logger.debug("Save: {0}".format(plotName))
# アニメーションGIF用に保持
png_lib.append(imageio.imread(plotName))
plt.close()
n += 1
logger.info(
"creating Gif {0}/movie_depth.gif, please Wait!".format(baseline_path))
imageio.mimsave('{0}/movie_depth.gif'.format(baseline_path),
png_lib,
fps=60)
# 終わったら後処理
cap.release()
cv2.destroyAllWindows()
logger.info("Done!!")
logger.info("深度推定結果: {0}".format(subdir))
