def test_difference(self):
_, image1, image2 = Loader.get_action('cube-1',
'2018-10-22-23-42-52-096',
['ed-v', 'ed-after'])
diff = image_difference(image1, image2)
imageio.imwrite(str(self.file_path / f'gen-diff.png'), diff.mat)
def get_mean(episode):
_, image = Loader.get_action(episode[0], episode[1]['id'], 'ed-after')
if image is None:
return {'id': episode[1]['id'], 'mean': 1e6}
return {'id': episode[1]['id'], 'mean': np.mean(image.mat)}
def test_heatmap(self):
_, image = Loader.get_action('cylinder-cube-1', '2019-03-26-09-51-08-827', 'ed-v')
if TEST_WITH_GPU:
model = Loader.get_model('cylinder-cube-1', 'model-6-arch-more-layer', output_layer='prob')
heatmapper = Heatmap(InferencePlanarPose, model, box=self.box)
heatmap = heatmapper.render(image)
imageio.imwrite(str(self.file_path / f'gen-heatmap.png'), heatmap)
def test_agent(self):
_, image = Loader.get_action('cylinder-cube-1', '2019-03-26-09-08-16-480', 'ed-v')
if TEST_WITH_GPU:
agent = Agent()
result = agent.infer([image], SelectionMethod.Max)
self.assertEqual(result.safe, True)
self.assertEqual(result.method, SelectionMethod.Max)
def test_agent_predict(self):
# 2019-03-11-14-56-07-284, 2019-03-14-11-26-17-352, 2019-03-12-16-14-54-658
_, image = Loader.get_action('cylinder-cube-1', '2019-03-11-14-56-07-284', 'ed-v')
if TEST_WITH_GPU:
prediction_model = Loader.get_model('cylinder-cube-1', 'predict-generator-3', custom_objects={'_one_hot': one_hot_gen(4)})
grasp_model = Loader.get_model('cylinder-cube-1', 'model-6-arch-more-layer', output_layer='prob')
shift_model = Loader.get_model('shifting', 'model-1', output_layer='prob')
agent = PredictAgent(prediction_model, grasp_model, shift_model)
agent.predict_actions([image], SelectionMethod.Top5, N=5, verbose=True)
def test_loader(self):
for suffix in ['ed-v', 'ed-side_b-0_400']:
action, image = Loader.get_action('cylinder-cube-1',
'2019-06-25-15-49-13-551',
suffix)
draw_around_box(image, box=Config.box)
imageio.imwrite(
str(self.file_path / f'gen-draw-around-box-{suffix}.png'),
image.mat)
self.assertEqual(image.mat.dtype, np.uint16)
self.assertEqual(image.pixel_size, 2000.0)
self.assertEqual(action.method, SelectionMethod.Prob)
def api_image(collection_name: str, episode_id: str, action_id: str,
suffix: str):
def send_image(image):
_, image_encoded = cv2.imencode('.jpg', image)
return flask.send_file(io.BytesIO(image_encoded),
mimetype='image/jpeg')
def send_empty_image():
empty = np.zeros((480, 752, 1))
cv2.putText(empty,
'?', (310, 300),
cv2.FONT_HERSHEY_SIMPLEX,
6,
100,
thickness=6)
return send_image(empty)
if flask.request.values.get('pose'):
action = Action(data=json.loads(flask.request.values.get('pose')))
image = Loader.get_image(collection_name,
episode_id,
int(action_id),
suffix,
images=action.images)
else:
try:
action, image = Loader.get_action(collection_name, episode_id,
int(action_id), suffix)
except Exception:
app.logger.warn('Could not find image:', collection_name,
episode_id, action_id, suffix)
return send_empty_image()
if suffix not in action.images.keys():
app.logger.warn(
f'Could not find suffix {collection_name}-{episode_id}-{action_id}-{suffix}'
)
return send_empty_image()
draw_pose(image, action.pose, convert_to_rgb=True)
# draw_pose(image, action.pose, convert_to_rgb=True, reference_pose=action.images[suffix]['pose'])
if flask.request.values.get('box', default=0, type=int):
draw_around_box(image, box=Config.box, draw_lines=True)
return send_image(image.mat / 255)
def print_before_after_image(episode_id: str):
action, before_image, after_image = Loader.get_action(
'shifting', episode_id, ['ed-v', 'ed-after'])
area_before_image = get_area_of_interest(before_image,
action.pose,
size_cropped=(300, 300),
size_result=(150, 150))
area_after_image = get_area_of_interest(after_image,
action.pose,
size_cropped=(300, 300))
white = [255 * 255] * 3
draw_line(area_before_image,
action.pose,
Affine(0, 0),
Affine(0.036, 0),
color=white,
thickness=2)
draw_line(area_before_image,
action.pose,
Affine(0.036, 0.0),
Affine(0.026, -0.008),
color=white,
thickness=2)
draw_line(area_before_image,
action.pose,
Affine(0.036, 0.0),
Affine(0.026, 0.008),
color=white,
thickness=2)
cv2.imwrite(
str(Path.home() / 'Desktop' / f'example-{episode_id}-before.png'),
area_before_image.mat)
cv2.imwrite(
str(Path.home() / 'Desktop' / f'example-{episode_id}-after.png'),
area_after_image.mat)
print('---')
print(episode_id)
def save_episode(predictor, database, episode_id, reward=1, action_type=1):
action, image, image_after = Loader.get_action(database, episode_id,
['ed-v', 'ed-after'])
draw_around_box(image, box=Config.box)
draw_around_box(image_after, box=Config.box)
# background_color = image.value_from_depth(get_distance_to_box(image, Config.box))
area = get_area_of_interest(image,
action.pose,
size_cropped=(256, 256),
size_result=predictor.size)
area_after = get_area_of_interest(image_after,
action.pose,
size_cropped=(256, 256),
size_result=predictor.size)
result = predictor.predict(area,
reward=reward,
action_type=action_type,
sampling=True,
number=20)
save_dir = Path.home() / 'Desktop' / 'predict-examples' / episode_id
save_dir.mkdir(exist_ok=True)
cv2.imwrite(str(save_dir / f'{predictor.name}_s_bef.png'), area.mat)
cv2.imwrite(str(save_dir / f'{predictor.name}_s_aft.png'), area_after.mat)
cv2.imwrite(str(save_dir / f'{predictor.name}_result.png'),
result[0] * 255)
if predictor.uncertainty:
result[result < 0.1] = np.nan
uncertainty = np.nanvar(result, axis=0)
uncertainty /= np.nanmax(uncertainty) * 0.25
uncertainty = np.clip(uncertainty * 255, 0, 255).astype(np.uint8)
uncertainty = cv2.applyColorMap(uncertainty, cv2.COLORMAP_JET)
cv2.imwrite(str(save_dir / f'{predictor.name}_unc.png'), uncertainty)
def api_image(episode_id):
def send_image(image):
_, image_encoded = cv2.imencode('.jpg', image)
return flask.send_file(io.BytesIO(image_encoded),
mimetype='image/jpeg')
def send_empty_image():
empty = np.zeros((480, 752, 1))
cv2.putText(empty,
'?', (310, 300),
cv2.FONT_HERSHEY_SIMPLEX,
6,
100,
thickness=6)
return send_image(empty)
database_name = flask.request.values.get('database')
suffix = flask.request.values.get('suffix')
if flask.request.values.get('pose'):
action = Action(data=json.loads(flask.request.values.get('pose')))
image = Loader.get_image(database_name,
episode_id,
suffix,
images=action.images)
else:
action, image = Loader.get_action(database_name, episode_id, suffix)
if not action or suffix not in action.images.keys() or not image:
return send_empty_image()
draw_pose(image,
action.pose,
convert_to_rgb=True,
reference_pose=action.images['ed-v']['pose'])
if int(flask.request.values.get('box', default=0)):
draw_around_box(image, box=Config.box, draw_lines=True)
return send_image(image.mat / 255)
type=str,
required=True)
parser.add_argument('-a', '--action', dest='action', type=int, default=0)
parser.add_argument('-s',
'--suffix',
dest='suffix',
type=str,
default='ed-v')
parser.add_argument('-m', '--model', dest='model', type=str)
parser.add_argument('-d', '--draw', action='store_true')
parser.add_argument('--area', action='store_true')
parser.add_argument('--convert', action='store_true')
parser.add_argument('--show', action='store_true')
args = parser.parse_args()
action, image = Loader.get_action(args.collection, args.episode,
args.action, args.suffix)
print('Action: ', action)
if args.area:
area_image = get_area_of_interest_new(image,
action.pose,
size_cropped=(200, 200))
if args.convert:
converter = Converter(grasp_z_offset=0.015, box=Config.box)
converter.grasp_convert(action, [image])
if args.show:
cv2.imshow('area_image', area_image.mat)
import os
from agents.agent import Agent
from data.loader import Loader
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = str(1)
agent = Agent()
data = []
for i, (d, e) in enumerate(Loader.yield_episodes('cylinder-cube-mc-1')):
action, image = Loader.get_action(d, e['id'], 'ed-v')
if not hasattr(action, 'estimated_reward'):
continue
data.append({
'id': e['id'],
# 'old': action.estimated_reward,
'new': agent.reward_for_action([image], action),
'reward': action.reward
})
sorted_data = sorted(data, key=lambda k: -abs(k['reward'] - k['new']))
for i, e in enumerate(sorted_data[:20]):
print(i, e)
from data.loader import Loader
data = []
for i, (d, e) in enumerate(Loader.yield_episodes('cube-1')):
action = Loader.get_action(d, e['id'])
if action.reward == 1 and action.final_pose and action.final_pose.d < 0.007:
print(d, e['id'])
from agents.agent_predict import Agent
from data.loader import Loader
from learning.utils.layers import one_hot_gen
from picking.param import SelectionMethod
# few objects (7-8): 2019-03-11-14-56-07-284
# many objects: 2019-07-01-14-03-11-150
# three cubes in a row: 2019-08-23-10-52-33-384
_, image = Loader.get_action('cylinder-cube-1', '2019-08-23-10-52-33-384',
'ed-v')
pred_model = Loader.get_model('cube-1', 'predict-bi-gen-5')
agent = Agent(pred_model)
# agent.plan_actions([image], SelectionMethod.Top5, depth=7, leaves=2, verbose=False)
# agent.plan_actions([image], SelectionMethod.Max, depth=7, leaves=1, verbose=False)
agent.plan_actions([image],
SelectionMethod.Top5,
depth=5,
leaves=3,
verbose=True)
from tensorflow.keras import Model # pylint: disable=E0401
from config import Config
from data.loader import Loader
from utils.heatmap import Heatmap
if __name__ == '__main__':
save_path = Path(__file__).parent.parent.parent / 'test' / 'generated'
collection = 'placing-3'
episode_id = '2020-02-04-00-34-54-455'
# episode_id = '2020-02-04-00-18-07-322'
model = Loader.get_model('placing-3-32-part-type-2')
action_grasp, image_grasp = Loader.get_action(collection, episode_id, 0,
'ed-v')
action_place, image_place, image_goal = Loader.get_action(
collection, episode_id, 1, ['ed-v', 'ed-goal'])
g = model.get_layer('grasp')
p = model.get_layer('place')
grasp_output = ['reward_grasp', 'z_m0']
place_output = ['reward_place', 'z_p']
grasp_model = Model(inputs=g.input,
outputs=[g.get_layer(l).output for l in grasp_output])
place_model = Model(inputs=p.input,
outputs=[p.get_layer(l).output for l in place_output])
indices = np.array([[22, 25, 18, 1], [13, 21, 17, 2], [7, 19, 16, 2],
import cv2
import numpy as np
from config import Config
from data.loader import Loader
from inference.planar import InferencePlanarPose
from picking.image import draw_pose
if __name__ == '__main__':
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = str(1)
np.set_printoptions(suppress=True)
action, image = Loader.get_action('cylinder-cube-mc-1', '2019-07-02-19-55-54-845', 'ed-v')
inference = InferencePlanarPose(
model=Loader.get_model('cylinder-cube-mc-1', 'model-1-mc'),
box=Config.box,
monte_carlo=160,
)
estimated_reward, estimated_reward_std = inference.infer_at_pose([image], action.pose)
print(estimated_reward, estimated_reward_std)
draw_pose(image, action.pose, convert_to_rgb=True)
cv2.imshow('image', image.mat)
cv2.waitKey(1500)
from pathlib import Path
import time
import cv2
from config import Config
from data.loader import Loader
from utils.image import draw_around_box, draw_pose, get_area_of_interest_new
if __name__ == '__main__':
lateral = False
suffix = 'ed-lateral_b-0_400' if lateral else 'ed-v'
action, image = Loader.get_action('placing-3', '2019-12-12-16-07-12-857',
0, 'ed-v')
# image = image.translate((0.0, 0.0, 0.05))
# image = image.rotate_x(-0.3, (0.0, 0.25))
draw_around_box(image, box=Config.box)
# draw_pose(image, action.pose, convert_to_rgb=True)
size_input = image.mat.shape[::-1]
size_cropped = (200, 200)
size_result = (32, 32)
scale = 4
image.mat = cv2.resize(image.mat,
(size_input[0] // scale, size_input[1] // scale))
image.pixel_size /= scale
s = time.time()
def test_inside_box(self):
action = Loader.get_action('cylinder-cube-1',
'2019-06-25-14-59-51-451')
conv = Converter(box=Config.box)
self.assertFalse(conv.is_pose_inside_box(action.pose))
# Loader.get_model('cylinder-cube-mc-1', 'model-1-mc', output_layer='prob'),
# box=Config.box,
# monte_carlo=160
# )
inference = InferencePlanarPose(
Loader.get_model('cylinder-cube-1',
'model-6-arch-more-layer',
output_layer='prob'),
box=Config.box,
)
# inference = InferencePlanarPose(
# Loader.get_model('shifting', 'model-3'),
# box=Config.box,
# )
_, image = Loader.get_action('cylinder-cube-mc-1',
'2019-07-02-13-27-22-246', 'ed-v')
indexer = GraspIndexer(gripper_classes=Config.gripper_classes)
converter = Converter(grasp_z_offset=Config.grasp_z_offset, box=Config.box)
times = []
for i in range(1):
start = time.time()
action = inference.infer([image], SelectionMethod.Top5, verbose=1)
indexer.to_action(action)
end = time.time()
times.append(end - start)
converter.calculate_pose(action, [image])
default='placing-eval')
parser.add_argument('-e',
'--episode',
dest='episode',
type=str,
required=True)
parser.add_argument('-ca',
'--camera',
dest='camera',
type=str,
default='ed')
parser.add_argument('--save', action='store_true')
args = parser.parse_args()
action, place_after, place_goal = Loader.get_action(
args.collection, args.episode, 1,
[args.camera + '-after', args.camera + '-goal'])
new_pose = RobotPose(action.pose)
if not args.save:
def update_image():
after_area = get_area_of_interest_new(place_after,
action.pose,
size_cropped=(200, 200))
goal_area = get_area_of_interest_new(place_goal,
new_pose,
size_cropped=(200, 200))
ontop = np.zeros((200, 200, 3), dtype=np.uint8)