def _get_saliency_matrix(self, position):
client = AWSClient.get_client('s3')
response = client.get_object(
Bucket=Constants.S3_BUCKETS['SALIENCY_MAPS'],
Key="{}_{}.json".format(self.hit_id, position))
data = json.load(response['Body'])
return np.array(data["saliencyMatrix"], np.uint8)
def run(self):
# Create DB session
session = TorchbearerDB.Session()
try:
for position in Constants.LANDMARK_POSITIONS.values():
if AWSClient.s3_key_exists(
Constants.S3_BUCKETS['STREETVIEW_IMAGES'],
"{}_{}.jpg".format(self.hit_id, position)):
# Load saliency mask and image from S3
sm = self._get_saliency_matrix()
img = self._get_streetview_image(position)
if os.environ.get('debug'):
plt.imshow(img, alpha=1)
plt.imshow(sm, alpha=0.6)
plt.show()
# Compute sum of saliency mask
image_saliency_sum = sm.sum()
# Get list of individual salient regions
masks = MaskMaker.get_masks_from_saliency_map(sm)
for mask in masks:
candidate = GrabCut.crop_image_with_saliency_mask(
img, mask)
# Tack a GUID onto candidate dict
candidate['id'] = uuid.uuid1()
candidate['position'] = position
# Compute visual saliency score
r = candidate['rect']
region_saliency_sum = sm[r['y1']:r['y2'],
r['x1']:r['x2']].sum()
candidate[
'visual_saliency_score'] = region_saliency_sum / float(
image_saliency_sum)
# candidate["image"].show()
# candidate["image_transparent"].show()
# Put cropped images into S3
self._put_cropped_images(candidate)
# Insert candidate landmark into DB
self._insert_candidate_landmark(candidate, session)
# Commit DB inserts
session.commit()
# Send success!
self.send_success()
except Exception as e:
traceback.print_exc()
session.rollback()
self.send_failure('MATRIX MASTER ERROR', e.message)
def run(self):
print("Starting mask task for ep {}, hit {}".format(
self.ep_id, self.hit_id))
# Create DB session
session = TorchbearerDB.Session()
try:
hit = session.query(Hit.Hit).filter_by(hit_id=self.hit_id).one()
hit.set_start_time_for_task("mask")
for position in Constants.LANDMARK_POSITIONS.values():
if AWSClient.s3_key_exists(
Constants.S3_BUCKETS['SALIENCY_MAPS'],
"{}_{}.json".format(self.hit_id, position)):
# Load saliency mask and image from S3
sm = self._get_saliency_matrix(position)
bounding_boxes = MaskMaker.make_bounding_boxes(sm)
for bb in bounding_boxes:
x1, x2, y1, y2 = [
bb[k] for k in ('x1', 'x2', 'y1', 'y2')
]
if os.environ.get('debug'):
cv2.imshow("Output", sm[y1:y2, x1:x2])
cv2.waitKey(0)
landmark = {
'id': uuid.uuid1(),
'rect': {
'x1': x1,
'x2': x2,
'y1': y1,
'y2': y2
},
'position': position
}
# Insert candidate landmark into DB
self._insert_candidate_landmark(landmark, session)
hit.set_end_time_for_task("mask")
# Commit DB inserts
session.commit()
# Send success!
self.send_success()
print("Completed mask task for ep {}, hit {}".format(
self.ep_id, self.hit_id))
except Exception as e:
traceback.print_exc()
session.rollback()
self.send_failure('MATRIX MASTER ERROR', e.message)
finally:
session.close()
def _get_streetview_image(self, position):
client = AWSClient.get_client('s3')
response = client.get_object(
Bucket=Constants.S3_BUCKETS['STREETVIEW_IMAGES'],
Key="{}_{}.jpg".format(self.hit_id, position))
img = Image.open(response['Body'])
# img.show()
return np.array(img)
def _delete_s3_objects(bucket, keys):
client = AWSClient.get_client('s3')
key_dict = dict(('Key', keyName) for keyName in keys)
try:
client.delete_objects(Bucket=bucket,
Delete={'Objects': [key_dict]})
except Exception:
print "Couldn't delete the given S3 object."
def _read_streetview_img_from_s3(self, position):
client = AWSClient.get_client('s3')
response = client.get_object(
Bucket=Constants.S3_BUCKETS['STREETVIEW_IMAGES'],
Key="{}_{}.jpg".format(self.ep_id, position)
)
img = Image.open(response['Body'])
# img.show()
img = np.array(img)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
return img
def _put_marked_streetview_image(img_file, landmark_id):
client = AWSClient.get_client('s3')
# Make sure file is "rewound"
img_file.seek(0)
# Put cropped image
client.put_object(
Body=img_file,
Bucket=Constants.S3_BUCKETS['MARKED_LANDMARK_IMAGES'],
Key="{0}.png".format(landmark_id))
def _put_cropped_image(img, landmark_id):
img_file = StringIO()
img.save(img_file, 'PNG')
client = AWSClient.get_client('s3')
# Put cropped image
client.put_object(
Body=img_file,
Bucket=Constants.S3_BUCKETS['TRANSPARENT_CROPPED_IMAGES'],
Key="{0}.png".format(landmark_id))
def _run_vision_task(self):
# Create DB session
session = TorchbearerDB.Session()
try:
for position in Constants.LANDMARK_POSITIONS.values():
if AWSClient.s3_key_exists(Constants.S3_BUCKETS['SALIENCY_MAPS'],
"{}_{}.json".format(self.hit_id, position)):
# Get streetview image for this Hit's ExecutionPoint
img = self._read_streetview_img_from_s3(position)
# Run object detection
detections = score_image(img)
# loop over detected objects, creating landmarks from them
for region in detections:
x1 = region['x1']
x2 = region['x2']
y1 = region['y1']
y2 = region['y2']
description = region['label']
visual_saliency_score = region['score']
if os.environ.get('debug'):
cv2.imshow("Output", img[y1:y2, x1:x2])
cv2.waitKey(0)
landmark = {
'id': uuid.uuid1(),
'description': description,
'rect': {'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2},
'position': position,
'visual_saliency_score': visual_saliency_score
}
# Insert candidate landmark into DB
self._insert_candidate_landmark(landmark, session)
# Commit DB inserts
session.commit()
self.send_success()
except Exception as e:
traceback.print_exc()
self.send_failure('OBJECT_DETECTION_ERROR', e.message)
def _put_cropped_images(candidate):
img_file = BytesIO()
transparent_img_file = BytesIO()
candidate['image'].save(img_file, 'PNG')
candidate['image_transparent'].save(transparent_img_file, 'PNG')
img_file.seek(0)
transparent_img_file.seek(0)
client = AWSClient.get_client('s3')
# Put cropped image
client.put_object(Body=img_file,
ContentType='image/png',
Bucket=Constants.S3_BUCKETS['CROPPED_IMAGES'],
Key="{0}.png".format(candidate['id']))
# Put transparent cropped image
client.put_object(
Body=transparent_img_file,
ContentType='image/png',
Bucket=Constants.S3_BUCKETS['TRANSPARENT_CROPPED_IMAGES'],
Key="{0}.png".format(candidate['id']))
def run(self):
print("Starting score task for ep {}, hit {}".format(
self.ep_id, self.hit_id))
# Create DB session
session = TorchbearerDB.Session()
try:
hit = session.query(Hit.Hit).filter_by(hit_id=self.hit_id).one()
hit.set_start_time_for_task("score")
# Load saliency mask and image from S3, across all positions available for this ExecutionPoint
for position in Constants.LANDMARK_POSITIONS.values():
if AWSClient.s3_key_exists(
Constants.S3_BUCKETS['STREETVIEW_IMAGES'],
"{}_{}.jpg".format(self.hit_id, position)):
sm = self._get_saliency_matrix(position)
img = self._get_streetview_image(position)
if os.environ.get('debug'):
plt.imshow(img, alpha=1)
plt.imshow(sm, alpha=0.6)
plt.show()
# Compute sum of saliency mask
image_saliency_sum = sm.sum()
# Retrieve landmarks for this hit and position
for landmark in session.query(Landmark.Landmark) \
.filter_by(hit_id=self.hit_id, position=position).all():
# Compute visual saliency score for this landmark
r = landmark.get_rect()
# If rectangle was not already defined by another service, do optimistic saliency search
# Note that this can still return None
# Save this rect with landmark back to db
if r is None:
r = OptimisticSearch.get_salient_area_at_degrees(
img, sm, landmark.relative_bearing)
landmark.set_rect(r)
visual_saliency_score = sm[r['y1']:r['y2'], r['x1']:r['x2']].sum() / float(image_saliency_sum) \
if r is not None else 0
landmark.visual_saliency_score = visual_saliency_score
hit.set_end_time_for_task("score")
# Commit DB inserts
session.commit()
# Send success!
self.send_success()
print("Completed score task for ep {}, hit {}".format(
self.ep_id, self.hit_id))
except Exception as e:
traceback.print_exc()
session.rollback()
self.send_failure('MATRIX MASTER ERROR', e.message)
finally:
session.close()
def _run_landmark_marker(self):
print("Starting mark task for ep {}, hit {}".format(
self.ep_id, self.hit_id))
# Create DB session
session = TorchbearerDB.Session()
try:
hit = session.query(Hit).filter_by(hit_id=self.hit_id).one()
hit.set_start_time_for_task("landmark_mark")
# Load from S3, across all positions available for corresponding ExecutionPoint
for position in Constants.LANDMARK_POSITIONS.values():
if AWSClient.s3_key_exists(
Constants.S3_BUCKETS['STREETVIEW_IMAGES'],
"{}_{}.jpg".format(self.hit_id, position)):
# Get streetview image for this Hit's ExecutionPoint
img_array = self._get_streetview_image(position)
# Get all landmarks for this hit for given position
for landmark in session.query(Landmark).filter_by(
hit_id=self.hit_id, position=position).all():
# Create figure and axes
fig, ax = plt.subplots(figsize=(8, 8), dpi=72)
# Remove axes
ax.axis('off')
fig.axes[0].get_xaxis().set_visible(False)
fig.axes[0].get_yaxis().set_visible(False)
# Add the image
ax.imshow(img_array)
# Create a Rectangle patch
x1 = landmark.get_rect()["x1"]
x2 = landmark.get_rect()["x2"]
y1 = landmark.get_rect()["y1"]
y2 = landmark.get_rect()["y2"]
width = x2 - x1
height = y2 - y1
rect = patches.Rectangle((x1, y1),
width,
height,
linewidth=2,
edgecolor='r',
facecolor='none')
# Add the patch to the Axes
ax.add_patch(rect)
# Save image
img_file = BytesIO()
fig.savefig(img_file,
format='png',
bbox_inches='tight',
pad_inches=0)
self._put_marked_streetview_image(
img_file, landmark.landmark_id)
# fig.show()
hit.set_end_time_for_task("landmark_mark")
# Commit DB inserts
session.commit()
self.send_success()
print("Completed mark task for ep {}, hit {}".format(
self.ep_id, self.hit_id))
except Exception, e:
traceback.print_exc()
self.send_failure('LANDMARK_MARK_ERROR', e.message)
def run(self):
# Create DB session
session = TorchbearerDB.Session()
try:
hit = session.query(Hit).filter_by(hit_id=self.hit_id).one()
hit.set_start_time_for_task("crop")
# Load from S3, across all positions available for corresponding ExecutionPoint
for position in Constants.LANDMARK_POSITIONS.values():
if AWSClient.s3_key_exists(
Constants.S3_BUCKETS['STREETVIEW_IMAGES'],
"{}_{}.jpg".format(self.hit_id, position)):
img = self._get_streetview_image(position)
# Load all Landmarks for this hit, position
for landmark in session.query(Landmark).filter_by(
hit_id=self.hit_id, position=position).all():
# Crop image according to Landmark rect
rect = landmark.get_rect()
rect = (rect['x1'], rect['y1'], rect['x2'], rect['y2'])
# Perform image segmentation to extract foreground object
arr = np.asarray(img)
cv_img = cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)
mask = np.zeros(cv_img.shape[:2], np.uint8)
bgdModel = np.zeros((1, 65), np.float64)
fgdModel = np.zeros((1, 65), np.float64)
cv2.grabCut(cv_img, mask, rect, bgdModel, fgdModel, 5,
cv2.GC_INIT_WITH_RECT)
# If mask==2 or mask== 1, mask2 get 0, other wise it gets 1 as 'uint8' type.
mask2 = np.where((mask == 2) | (mask == 0), 0,
1).astype('uint8')
# Project image into RGBA space
cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2BGRA)
# Set alpha based on segment_mask.
# Set red channel to 100% for viewers that don't support alpha channel.
cv_img[mask2 == 0] = [255, 0, 0, 0]
# Convert cv2 img back to PIL img
img = Image.fromarray(cv_img)
# Crop cut image down to landmark rect
img = img.crop(rect)
if os.environ.get('debug'):
plt.imshow(img)
plt.show()
# Put cropped images into S3
self._put_cropped_image(img, landmark.landmark_id)
hit.set_end_time_for_task("crop")
# Commit DB inserts, if any
session.commit()
# Send success!
self.send_success()
except Exception as e:
traceback.print_exc()
session.rollback()
self.send_failure('CROP_ERROR', e.message)
finally:
session.close()