class WhaleDataset(Dataset):
def __init__(self, train_val_cvs_path, train,
bbox_transform=None,
image_transform=None):
super().__init__()
self.train_folds_path = train_val_cvs_path
self.train = train
self.bbox_transform = bbox_transform
self.image_transform = image_transform
self.turbo_jpeg = TurboJPEG('/usr/lib/x86_64-linux-gnu/libturbojpeg.so.0')
self.images, self.class_indexes, self.bboxes, self.id2class_idx = \
get_samples(train_val_cvs_path, train)
def __len__(self):
return len(self.images)
def __getitem__(self, idx):
image = self.images[idx]
class_index = self.class_indexes[idx]
bbox = self.bboxes[idx]
image = self.turbo_jpeg.decode(image)
if self.bbox_transform is not None:
image, bbox = self.bbox_transform(image, bbox)
if self.image_transform is not None:
image = self.image_transform(image)
class_index = torch.tensor(class_index)
return image, class_index
def do_mem_profile_decompress(): jpeg = TurboJPEG() out_img = 'jpegs/single_v0.jpg' in_file = open(out_img, 'rb') bgr_array = jpeg.decode(in_file.read()) in_file.close() return None
def read_image_turbo_jpeg(path: str,
jpeg_reader: TurboJPEG = None) -> np.array:
if not jpeg_reader:
jpeg_reader = TurboJPEG()
file = open(path, "rb")
image = jpeg_reader.decode(file.read(), 1) # 0 - RGB, 1 - BGR
return image
class TurboImageLoader(BaseImageLoader):
def __init__(self, **kwargs):
self.jpeg = TurboJPEG(**kwargs)
def load(self, img_path):
with open(img_path, 'rb') as in_file:
img = self.jpeg.decode(in_file.read(), TJPF_RGB)
return img
class TurboJpegHandler(object):
"""The object handling JPEG compression/decompression"""
def __init__(self, jpeg_quality):
self.jpeg_quality = jpeg_quality
self.jpeg = TurboJPEG()
def compress(self, cv2_img):
return self.jpeg.encode(cv2_img, quality=self.jpeg_quality)
def decompress(self, img_buffer):
return self.jpeg.decode(img_buffer)
def do_benchmarking_decompress(in_img): timeval = [] jpeg = TurboJPEG() for j in range(100): for i in range(100): in_file = open(in_img, 'rb') start = time.process_time() bgr_array = jpeg.decode(in_file.read()) end = time.process_time() val = (end - start)*1000 #msec timeval.append(val) in_file.close() return timeval
def test_turbo():
img = np.load(op.join(op.dirname(__file__), 'data', 'mona_lisa.npy'))
turbo = TurboJPEG()
encoded = turbo.encode(img,
quality=95,
pixel_format=TJPF.BGR,
jpeg_subsample=TJSAMP.YUV420)
assert len(img.data) > len(encoded)
assert encoded == turbo.encode(img,
quality=95,
pixel_format=TJPF.BGR,
jpeg_subsample=TJSAMP.YUV420)
assert turbo.info(encoded) == (341, 229, 'YUV420', 'BGR')
decoded = turbo.decode(encoded)
np.testing.assert_equal(decoded, turbo.decode(encoded))
assert not np.array_equal(
decoded, turbo.decode(encoded, fast_dct=True, fast_upsample=False))
assert not np.array_equal(
decoded, turbo.decode(encoded, fast_dct=False, fast_upsample=True))
assert not np.array_equal(
decoded, turbo.decode(encoded, fast_dct=True, fast_upsample=True))
assert phash_compare(img, decoded) <= 5
def decode_img(buffer):
if TurboJPEG is not None:
global g_jpeg
if g_jpeg is None:
g_jpeg = TurboJPEG()
img = g_jpeg.decode(buffer, TJCS_RGB)
if img.shape[-1] == 1:
img = img[:, :, 0]
return img
buff = io.BytesIO(buffer)
img = PIL.Image.open(buff)
img = pillow2array(img, 'color')
return img
def unpack_4fr(file_path):
jpeg_encoder = TurboJPEG('turbojpeg.dll')
with open(file_path, 'rb') as f:
data = f.read()
header = data[:8]
geo_size, texture_size = struct.unpack('II', header)
geo_buffer = data[8:8 + geo_size]
texture_buffer = data[8 + geo_size:]
# obj
geo_buffer = lz4framed.decompress(geo_buffer)
point_list = np.frombuffer(geo_buffer, dtype=np.float32)
point_list = point_list.reshape(-1, 5)
pos_list = point_list[:, 0:3]
uv_list = point_list[:, 3:5]
uv_list = np.array(uv_list, np.float32)
uv_list -= [0, 1.0]
uv_list *= [1, -1]
pos_strings = [f'v {x} {y} {z}' for x, y, z in pos_list]
uv_strings = [f'vt {u} {v}' for u, v in uv_list]
face_strings = [
f'f {f}/{f} {f + 1}/{f + 1} {f + 2}/{f + 2}'
for f in range(1, point_list.shape[0], 3)
]
obj_data = ['g'] + pos_strings + uv_strings + ['g'] + face_strings
obj_data = '\n'.join(obj_data)
with open(file_path.replace('4dr', 'obj'), 'w') as f:
f.write(obj_data)
# jpg
with open(file_path.replace('4dr', 'jpg'), 'wb') as f:
im = jpeg_encoder.decode(texture_buffer, TJPF_RGB)
f.write(jpeg_encoder.encode(im))
return file_path
class RandomWhaleDataset(Dataset):
def __init__(self, train_val_cvs_path, train,
balance_coef=0,
size=20000,
bbox_transform=None,
image_transform=None):
super().__init__()
self.train_folds_path = train_val_cvs_path
self.train = train
self.balance_coef = balance_coef
self.size = size
self.bbox_transform = bbox_transform
self.image_transform = image_transform
self.turbo_jpeg = TurboJPEG('/usr/lib/x86_64-linux-gnu/libturbojpeg.so.0')
self.id2samples, self.id2count, self.id2class_idx = \
get_random_samples(train_val_cvs_path, train)
self.id2prob = get_balanced_probs(self.id2count, balance_coef)
self.whale_id_lst = list(self.id2samples.keys())
self.prob_lst = [self.id2prob[i] for i in self.whale_id_lst]
def __len__(self):
return self.size
def __getitem__(self, idx):
seed = int(time.time() * 1000.0) + idx
random.seed(seed)
np.random.seed(seed % (2**32 - 1))
whale_id = np.random.choice(self.whale_id_lst, p=self.prob_lst)
image, class_index, bbox = random.choice(self.id2samples[whale_id])
image = self.turbo_jpeg.decode(image)
if self.bbox_transform is not None:
image, bbox = self.bbox_transform(image, bbox)
if self.image_transform is not None:
image = self.image_transform(image)
class_index = torch.tensor(class_index)
return image, class_index
class TurboJpegLoader(ImageLoader):
def __init__(self, path, **kwargs):
super(TurboJpegLoader, self).__init__(path, **kwargs)
self.jpeg_reader = TurboJPEG(
) # create TurboJPEG object for image reading
def __next__(self):
start = timer()
file = open(self.dataset[self.sample_idx],
"rb") # open the input file as bytes
full_time = timer() - start
if self.mode == "RGB":
mode = 0
elif self.mode == "BGR":
mode = 1
start = timer()
image = self.jpeg_reader.decode(file.read(), mode) # decode raw image
full_time += timer() - start
self.sample_idx += 1
return image, full_time
def decode():
jpeg = TurboJPEG()
image_folder = '/home/matrix/data/val/'
cnt = 0
time_sum = 0.0
for fname in sorted(os.listdir(image_folder)):
fpath = os.path.join(image_folder, fname)
# print(fpath)
in_file = open(fpath, 'rb')
jpg = in_file.read()
cnt += 1
# (width, height, jpeg_subsample, jpeg_colorspace) = jpeg.decode_header(jpg)
# print(width, height, jpeg_subsample, jpeg_colorspace)
begin = time.time() * 1000
raw = jpeg.decode(jpg)
end = time.time() * 1000
time_sum += end - begin
in_file.close()
print("image cnt: ", cnt)
print("time per image is(ms):", time_sum / cnt)
def main():
input_img = "rabbit.jpeg"
output_name = 'output.jpg'
jpeg = TurboJPEG()
# print("=============Read Image=============")
img = cv2.imread(input_img)
with open(input_img, 'rb') as infile:
img = jpeg.decode(infile.read())
# print("=============Write Image=============")
cv2.imwrite(output_name, img)
with open(output_name, 'wb') as outfile:
outfile.write(jpeg.encode(img, quality=30))
base1 = jpeg.encode(img, quality=30)
base64.b64encode(base1).decode("ascii")
# print("=============Python Utils=============")
base2 = compress_encode_image(img)
decode_decompress_image(base2)
class ImageLoader(mp.Process):
def __init__(self, idx, worker_stats, filename_queue, color_buffer_queue,
*args, **kwargs):
super().__init__(*args, **kwargs)
self.idx = idx
self.worker_stats = worker_stats
self.parser = ImageParser(idx, worker_stats)
self.jpeg_loader = TurboJPEG()
self.num_images = 0
self.file_size_sum = 0
self.pixel_sum = 0
self.filename_queue = filename_queue
self.color_buffer_queue = color_buffer_queue
self.is_running = mp.Value(ctypes.c_bool, True)
def read_png(self, buf):
x = np.frombuffer(buf, dtype=np.uint8)
img_np = cv2.imdecode(x, cv2.IMREAD_UNCHANGED)
if img_np is not None:
if img_np.dtype == np.uint16 and img_np.max() > 255:
img_np = (img_np // 256).astype(np.uint8)
return img_np
def read_jpeg(self, buf):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return self.jpeg_loader.decode(buf)
def read_image(self, filename, buf):
if filename.endswith(".png"):
bgr_array = self.read_png(buf)
else:
try:
bgr_array = self.read_jpeg(buf)
except OSError:
bgr_array = None
if bgr_array is not None:
if len(bgr_array.shape) > 2 and bgr_array.shape[2] == 4:
# return None
# print("need to realign memory")
bgr_array = np.ascontiguousarray(bgr_array[:, :, :3])
if len(bgr_array.shape) == 2:
new_array = np.zeros(bgr_array.shape + (3, ), dtype=np.uint8)
for i in range(3):
new_array[:, :, i] = bgr_array
bgr_array = new_array
# print(bgr_array.shape)
return bgr_array
def print_stats(self, i, t0, t1, cl0, cl1):
mp = self.pixel_sum / 1024**2
mb = self.file_size_sum / 1024**2
mp_per_second = mp / (t1 - t0)
mb_per_second = mb / (t1 - t0)
print(f"\r{i:4d}", end="\t", flush=True)
print(f"{mp_per_second:8.1f}MP/s", end="\t", flush=True)
print(f"{mb_per_second:.2f}MB/s", end="\t")
print(f"({(cl1-cl0) * 1e3:6.1f}ms) ({mp:7.1f}MP)", end="")
def load_single_image(self, filename, in_file):
# cl0 = clock()
buf = in_file.read()
bgr_array = self.read_image(filename, buf)
if bgr_array is None:
return
assert bgr_array.dtype == np.uint8
self.parser.add_image(bgr_array)
# self.parser_queue.put(bgr_array)
# self.image_parser.add_image(bgr_array)
# cl1 = clock()
self.file_size_sum += os.path.getsize(directory + filename)
self.pixel_sum += bgr_array.size // 3
# print(f"{filename} parsed")
# t1 = time.time()
# self.print_stats(i, t0, t1, cl0, cl1)
def run(self):
self.parser.compile()
while self.is_running.value:
# while True:
try:
image_data = self.filename_queue.get(True, 1)
except queue.Empty:
continue
filename = f"{image_data['filename']}.{image_data['filetype']}"
with open(directory + filename, 'rb') as in_file:
self.load_single_image(filename, in_file)
# time.sleep(1)
# print(f"Completed {image_data}")
self.parser.finalize_parser()
self.color_buffer_queue.put(self.parser.col_buffer)
class ImageFolder(torchvision.datasets.ImageFolder):
"""A generic data loader where the images are arranged in this way: ::
root/dog/xxx.png
root/dog/xxy.png
root/dog/[...]/xxz.png
root/cat/123.png
root/cat/nsdf3.png
root/cat/[...]/asd932_.png
Args:
root (string): Root directory path.
transform (callable, optional): A function/transform that takes in an PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
target_transform (callable, optional): A function/transform that takes in the
target and transforms it.
Attributes:
classes (list): List of the class names sorted alphabetically.
class_to_idx (dict): Dict with items (class_name, class_index).
imgs (list): List of (image path, class_index) tuples
"""
def __init__(
self,
root: str,
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None
):
super(ImageFolder, self).__init__(root, transform, target_transform)
self.jpeg: Optional[TurboJPEG] = None
def read_image_to_bytes(self, path: str):
fd = open(path, 'rb')
img_str = fd.read()
fd.close()
return img_str
def decode_img_libjpeg_turbo(self, img_str: str):
if self.jpeg is None:
self.jpeg = TurboJPEG(lib_path=local_libturbo_path)
bgr_array = self.jpeg.decode(img_str)
return bgr_array
def __getitem__(self, idx: int):
path = self.imgs[idx][0]
label = self.imgs[idx][1]
if path.endswith(".jpg") or path.endswith(".jpeg"):
img_str = self.read_image_to_bytes(path)
img = self.decode_img_libjpeg_turbo(img_str)
else:
img = cv2.imread(path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if self.transform:
img = self.transform(img)
img = img.numpy()
return img.astype('float32'), label
self.url = f'{self.ip}:{self.port}/test/'
self.test_url = f'{self.ip}:{self.port}/test/'
@profile
def encode(img: np.ndarray):
pass
def decode():
pass
if __name__ == '__main__':
cfg = CFG()
jpeg = TurboJPEG()
input_img = 'rabbit.jpeg'
with open(input_img, 'rb') as infile:
bgr_array = jpeg.decode(infile.read())
img_enc = jpeg.encode(bgr_array,quality=20)
print(type(img_enc))
print(sys.getsizeof(bgr_array))
print(sys.getsizeof(img_enc))
print(cfg.url)
try:
response = requests.post(cfg.url,data=img_enc)
if response.status_code==200:
print("Success")
else:
raise(response.status_code)
except:
class TTNet_Dataset(Dataset):
def __init__(self,
events_infor,
org_size,
input_size,
transform=None,
num_samples=None):
self.events_infor = events_infor
self.w_org = org_size[0]
self.h_org = org_size[1]
self.w_input = input_size[0]
self.h_input = input_size[1]
self.w_resize_ratio = self.w_org / self.w_input
self.h_resize_ratio = self.h_org / self.h_input
self.transform = transform
if num_samples is not None:
self.events_infor = self.events_infor[:num_samples]
def __len__(self):
return len(self.events_infor)
def __resize_ball_pos__(self, ball_pos_xy, w_ratio, h_ratio):
return np.array([ball_pos_xy[0] / w_ratio, ball_pos_xy[1] / h_ratio])
def __check_ball_pos__(self, ball_pos_xy, w, h):
if not ((0 < ball_pos_xy[0] < w) and (0 < ball_pos_xy[1] < h)):
ball_pos_xy[0] = -1.
ball_pos_xy[1] = -1.
def __getitem__(self, index):
img_path_list, org_ball_pos_xy, target_events, seg_path = self.events_infor[
index]
# Load segmentation
seg_img = load_raw_img(seg_path)
self.jpeg_reader = TurboJPEG(
) # improve it later (Only initialize it once)
# Load a sequence of images (-4, 4), resize images before stacking them together
# Use TurboJPEG to speed up the loading images' phase
resized_imgs = []
for img_path in img_path_list:
in_file = open(img_path, 'rb')
resized_imgs.append(
cv2.resize(self.jpeg_reader.decode(in_file.read(), 0),
(self.w_input, self.h_input)))
in_file.close()
resized_imgs = np.dstack(resized_imgs) # (128, 320, 27)
# Adjust ball pos: full HD --> (320, 128)
global_ball_pos_xy = self.__resize_ball_pos__(org_ball_pos_xy,
self.w_resize_ratio,
self.h_resize_ratio)
# Apply augmentation
if self.transform:
resized_imgs, global_ball_pos_xy, seg_img = self.transform(
resized_imgs, global_ball_pos_xy, seg_img)
# Adjust ball pos: (320, 128) --> full HD
org_ball_pos_xy = self.__resize_ball_pos__(global_ball_pos_xy,
1. / self.w_resize_ratio,
1. / self.h_resize_ratio)
# If the ball position is outside of the resized image, set position to -1, -1 --> No ball (just for safety)
self.__check_ball_pos__(org_ball_pos_xy, self.w_org, self.h_org)
self.__check_ball_pos__(global_ball_pos_xy, self.w_input, self.h_input)
# Transpose (H, W, C) to (C, H, W) --> fit input of Pytorch model
resized_imgs = resized_imgs.transpose(2, 0, 1)
target_seg = seg_img.transpose(2, 0, 1).astype(np.float)
# Segmentation mask should be 0 or 1
target_seg[target_seg < 75] = 0.
target_seg[target_seg >= 75] = 1.
return resized_imgs, org_ball_pos_xy.astype(np.int), global_ball_pos_xy.astype(np.int), \
target_events, target_seg
class AprilTagDetector(DTROS):
def __init__(self):
super(AprilTagDetector, self).__init__(
node_name='apriltag_detector_node',
node_type=NodeType.PERCEPTION
)
# get static parameters
self.family = rospy.get_param('~family', 'tag36h11')
self.ndetectors = rospy.get_param('~ndetectors', 1)
self.nthreads = rospy.get_param('~nthreads', 1)
self.quad_decimate = rospy.get_param('~quad_decimate', 1.0)
self.quad_sigma = rospy.get_param('~quad_sigma', 0.0)
self.refine_edges = rospy.get_param('~refine_edges', 1)
self.decode_sharpening = rospy.get_param('~decode_sharpening', 0.25)
self.tag_size = rospy.get_param('~tag_size', 0.065)
self.rectify_alpha = rospy.get_param('~rectify_alpha', 0.0)
# dynamic parameter
self.detection_freq = DTParam(
'~detection_freq',
default=-1,
param_type=ParamType.INT,
min_value=-1,
max_value=30
)
self._detection_reminder = DTReminder(frequency=self.detection_freq.value)
# camera info
self._camera_parameters = None
self._mapx, self._mapy = None, None
# create detector object
self._detectors = [Detector(
families=self.family,
nthreads=self.nthreads,
quad_decimate=self.quad_decimate,
quad_sigma=self.quad_sigma,
refine_edges=self.refine_edges,
decode_sharpening=self.decode_sharpening
) for _ in range(self.ndetectors)]
self._renderer_busy = False
# create a CV bridge object
self._jpeg = TurboJPEG()
# create subscribers
self._img_sub = rospy.Subscriber(
'~image',
CompressedImage,
self._img_cb,
queue_size=1,
buff_size='20MB'
)
self._cinfo_sub = rospy.Subscriber(
'~camera_info',
CameraInfo,
self._cinfo_cb,
queue_size=1
)
# create publisher
self._tag_pub = rospy.Publisher(
'~detections',
AprilTagDetectionArray,
queue_size=1,
dt_topic_type=TopicType.PERCEPTION,
dt_help='Tag detections',
)
self._img_pub = rospy.Publisher(
'~detections/image/compressed',
CompressedImage,
queue_size=1,
dt_topic_type=TopicType.VISUALIZATION,
dt_help='Camera image with tag publishs superimposed',
)
# create thread pool
self._workers = ThreadPoolExecutor(self.ndetectors)
self._tasks = [None] * self.ndetectors
# create TF broadcaster
self._tf_bcaster = tf.TransformBroadcaster()
def on_shutdown(self):
self.loginfo('Shutting down workers pool')
self._workers.shutdown()
def _cinfo_cb(self, msg):
# create mapx and mapy
H, W = msg.height, msg.width
# create new camera info
self.camera_model = PinholeCameraModel()
self.camera_model.fromCameraInfo(msg)
# find optimal rectified pinhole camera
with self.profiler('/cb/camera_info/get_optimal_new_camera_matrix'):
rect_K, _ = cv2.getOptimalNewCameraMatrix(
self.camera_model.K,
self.camera_model.D,
(W, H),
self.rectify_alpha
)
# store new camera parameters
self._camera_parameters = (rect_K[0, 0], rect_K[1, 1], rect_K[0, 2], rect_K[1, 2])
# create rectification map
with self.profiler('/cb/camera_info/init_undistort_rectify_map'):
self._mapx, self._mapy = cv2.initUndistortRectifyMap(
self.camera_model.K,
self.camera_model.D,
None,
rect_K,
(W, H),
cv2.CV_32FC1
)
# once we got the camera info, we can stop the subscriber
self.loginfo('Camera info message received. Unsubscribing from camera_info topic.')
# noinspection PyBroadException
try:
self._cinfo_sub.shutdown()
except BaseException:
pass
def _detect(self, detector_id, msg):
# turn image message into grayscale image
with self.profiler('/cb/image/decode'):
img = self._jpeg.decode(msg.data, pixel_format=TJPF_GRAY)
# run input image through the rectification map
with self.profiler('/cb/image/rectify'):
img = cv2.remap(img, self._mapx, self._mapy, cv2.INTER_NEAREST)
# detect tags
with self.profiler('/cb/image/detection'):
tags = self._detectors[detector_id].detect(
img, True, self._camera_parameters, self.tag_size)
# pack detections into a message
tags_msg = AprilTagDetectionArray()
tags_msg.header.stamp = msg.header.stamp
tags_msg.header.frame_id = msg.header.frame_id
for tag in tags:
# turn rotation matrix into quaternion
q = _matrix_to_quaternion(tag.pose_R)
p = tag.pose_t.T[0]
# create single tag detection object
detection = AprilTagDetection(
transform=Transform(
translation=Vector3(
x=p[0],
y=p[1],
z=p[2]
),
rotation=Quaternion(
x=q[0],
y=q[1],
z=q[2],
w=q[3]
)
),
tag_id=tag.tag_id,
tag_family=str(tag.tag_family),
hamming=tag.hamming,
decision_margin=tag.decision_margin,
homography=tag.homography.flatten().astype(np.float32).tolist(),
center=tag.center.tolist(),
corners=tag.corners.flatten().tolist(),
pose_error=tag.pose_err
)
# add detection to array
tags_msg.detections.append(detection)
# publish tf
self._tf_bcaster.sendTransform(
p.tolist(),
q.tolist(),
msg.header.stamp,
'tag/{:s}'.format(str(tag.tag_id)),
msg.header.frame_id
)
# publish detections
self._tag_pub.publish(tags_msg)
# update healthy frequency metadata
self._tag_pub.set_healthy_freq(self._img_sub.get_frequency())
self._img_pub.set_healthy_freq(self._img_sub.get_frequency())
# render visualization (if needed)
if self._img_pub.anybody_listening() and not self._renderer_busy:
self._renderer_busy = True
Thread(target=self._render_detections, args=(msg, img, tags)).start()
def _img_cb(self, msg):
# make sure we have received camera info
if self._camera_parameters is None:
return
# make sure we have a rectification map available
if self._mapx is None or self._mapy is None:
return
# make sure somebody wants this
if (not self._img_pub.anybody_listening()) and (not self._tag_pub.anybody_listening()):
return
# make sure this is a good time to detect (always keep this as last check)
if not self._detection_reminder.is_time(frequency=self.detection_freq.value):
return
# make sure we are still running
if self.is_shutdown:
return
# ---
# find the first available worker (if any)
for i in range(self.ndetectors):
if self._tasks[i] is None or self._tasks[i].done():
# submit this image to the pool
self._tasks[i] = self._workers.submit(self._detect, i, msg)
break
def _render_detections(self, msg, img, detections):
with self.profiler('/publishs_image'):
# get a color buffer from the BW image
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# draw each tag
for detection in detections:
for idx in range(len(detection.corners)):
cv2.line(
img,
tuple(detection.corners[idx - 1, :].astype(int)),
tuple(detection.corners[idx, :].astype(int)),
(0, 255, 0)
)
# draw the tag ID
cv2.putText(
img,
str(detection.tag_id),
org=(
detection.corners[0, 0].astype(int) + 10,
detection.corners[0, 1].astype(int) + 10
),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=0.8,
color=(0, 0, 255)
)
# pack image into a message
img_msg = CompressedImage()
img_msg.header.stamp = msg.header.stamp
img_msg.header.frame_id = msg.header.frame_id
img_msg.format = 'jpeg'
img_msg.data = self._jpeg.encode(img)
# ---
self._img_pub.publish(img_msg)
self._renderer_busy = False
class RSNADataset(Dataset):
def __init__(
self,
csv_path: str,
img_dir: str,
file_extension: str = 'dcm',
mode: str = 'train',
fold: int = 0,
k_fold: int = 5,
transform=None,
network_type: str = 'cnn',
max_sequence: int = 1083,
):
assert mode in ['train', 'val', 'test']
assert network_type in ['cnn', 'rnn', 'cnn_rnn']
assert -1 <= fold < 5
assert 15 % k_fold == 0
self.transform = transform
self.csv_path = Path(csv_path)
self.img_dir = Path(img_dir)
self.file_extension = file_extension
self.mode = mode
self.network_type = network_type
self.max_sequence = max_sequence
if network_type == 'cnn':
self.target_cols = [
'pe_present_on_image',
]
elif network_type == 'rnn':
self.target_cols = [
'negative_exam_for_pe',
'indeterminate',
'chronic_pe', 'acute_and_chronic_pe', # not indeterminate. Only One is true.
'central_pe', 'leftsided_pe', 'rightsided_pe', # not indeterminate. At least One is true.
'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1', # not indeterminate. Only One is true.
'pe_present_on_image',
]
if self.file_extension == 'jpg':
self.jpeg_reader = TurboJPEG()
df = pd.read_csv(self.csv_path)
df["file_name"] = df.SOPInstanceUID + '.' + self.file_extension
if self.file_extension != 'dcm':
df.z_pos_order = df.z_pos_order.map(lambda x: f'{x:04}')
df.file_name = df.z_pos_order + '_' + df.file_name
df["image_name"] = str(self.img_dir) + '/' + \
df.StudyInstanceUID + '/' + df.SeriesInstanceUID + '/' + df.file_name
self.df = df if fold == -1 else self._make_fold(df, fold, k_fold, mode=mode)
if self.network_type == 'rnn' or self.network_type == 'cnn_rnn':
self.df["path_to_series_id"] = str(self.img_dir) + '/' + \
self.df.StudyInstanceUID + '/' + self.df.SeriesInstanceUID
self.path_to_series_id = self.df["path_to_series_id"].unique()
def __len__(self):
if self.network_type == 'cnn':
return len(self.df)
elif self.network_type == 'rnn':
return len(self.path_to_series_id)
elif self.network_type == 'cnn_rnn':
return len(self.path_to_series_id)
def __getitem__(self, index):
if self.network_type == 'cnn':
return self._get_single_image(index)
else:
return self._get_series(index)
def _get_single_image(self, index):
data = self.df.iloc[index]
return self._get_img_label(data)
def _get_series(self, index):
if self.network_type == 'rnn':
return self._read_embeddings(index)
elif self.network_type == 'cnn_rnn':
if self.file_extension == 'npz':
return self._read_series_npz(index)
else:
return self._read_series_images(index)
def _read_embeddings(self, index):
data_path = self.path_to_series_id[index]
data_path = Path(data_path).with_suffix('.npz')
data = np.load(data_path)
embeddings = data['embeddings']
labels = data['labels']
sequence_length, _ = embeddings.shape
embeddings = self._padding_sequence(sequence_length, embeddings, 0)
labels = self._padding_sequence(sequence_length, labels, -1)
return embeddings, labels, sequence_length
def _read_series_npz(self, index):
data_path = Path(self.path_to_series_id[index]).with_suffix('.npz')
data = np.load(data_path)
imgs = data['imgs'] #(sequence, 3, h, w)
labels = data['labels'] #(sequence, n_class)
if self.transform is not None:
imgs = imgs.transpose(0, 2, 3, 1)
imgs = [self.transform(image=img).transpose(
2, 0, 1) for img in imgs]
imgs = np.stack(imgs)
imgs = imgs.astype('float32')
labels = labels.astype('float32')
return imgs, labels
def _read_series_images(self, index):
# use only when inference.
data_path = self.path_to_series_id[index]
dicoms, dicom_files = self._load_dicom_array(data_path)
imgs = self._get_three_windowing_image(dicoms)
if self.transform is not None:
imgs = imgs.transpose(0, 2, 3, 1)
imgs = [self.transform(image=img).transpose(
2, 0, 1) for img in imgs]
imgs = np.stack(imgs)
imgs = imgs.astype('float32')
exam_level_name, image_level_name = self._get_file_names(dicom_files)
return imgs, exam_level_name, image_level_name
def _get_img_label(self, data):
if self.file_extension == 'jpg':
binary = open(data.image_name, "rb")
img = self.jpeg_reader.decode(binary.read(), 0)
elif self.file_extension == 'dcm':
raise NotImplementedError
if self.transform is not None:
img = self.transform(image=img)
img = img.transpose(2, 0, 1).astype('float32')
labels = data[self.target_cols].values.astype('float32')
return img, labels
def _make_fold(self, df, fold, k_fold, mode='train'):
df_new = df.copy()
offset = 15 // k_fold
target = [i + fold * offset for i in range(offset)]
if mode == 'train':
df_new = df_new.query(f'fold not in {target}')
else:
df_new = df_new.query(f'fold in {target}')
return df_new
def _padding_sequence(self, sequence_length, target, value):
pad_len = self.max_sequence - sequence_length
assert pad_len >= 0
if pad_len > 0:
padding = [np.full_like(target[0], value)] * pad_len
target = np.concatenate([target, padding])
return target
def _load_dicom_array(self, path_to_series_id):
dicom_files = list(Path(path_to_series_id).glob('*.dcm'))
dicoms = [pydicom.dcmread(d) for d in dicom_files]
slope = float(dicoms[0].RescaleSlope)
intercept = float(dicoms[0].RescaleIntercept)
# Assume all images are axial
z_pos = [float(d.ImagePositionPatient[-1]) for d in dicoms]
dicoms = np.asarray([d.pixel_array for d in dicoms])
dicoms = dicoms[np.argsort(z_pos)]
dicoms = dicoms * slope
dicoms = dicoms + intercept
dicom_files = np.array(dicom_files)[np.argsort(z_pos)]
return dicoms, dicom_files
def _windowing(self, img, window_length, window_width):
upper = window_length + window_width // 2
lower = window_length - window_width // 2
x = np.clip(img.copy(), lower, upper)
x = x - np.min(x)
x = x / np.max(x)
x = (x * 255.0).astype('uint8')
return x
def _get_three_windowing_image(self, dicoms):
img_lung = np.expand_dims(
self._windowing(dicoms, -600, 1500), axis=1)
img_mediastinal = np.expand_dims(
self._windowing(dicoms, 40, 400), axis=1)
img_pe_specific = np.expand_dims(
self._windowing(dicoms, 100, 700), axis=1)
return np.concatenate([
img_lung, img_pe_specific, img_mediastinal], axis=1)
def _get_file_names(self, dicom_files):
exam_level_name = str(dicom_files[0].parent.parent.stem)
dicom_files = dicom_files.tolist()
image_level_name = list(map(lambda x: str(x.stem), dicom_files))
return exam_level_name, image_level_name
from PIL import Image
from time import perf_counter
import numpy as np
from mmap import mmap
mm = mmap(-1, 2**24, tagname='SharedMemory')
def timing(n, f, initial):
t = 0
for i in range(n):
initial()
start = perf_counter()
f(i)
t += perf_counter() - start
return t
initial = lambda: mm.seek(0)
jpeg = TurboJPEG("C:/Users/lotress/MoePhoto/test/libturbojpeg.dll")
filePath = r'C:\Users\lotress\Documents\福州轨道交通线路图（2050+）@chinho.jpg'
imgFile = open(filePath, 'rb')
imgBuf = imgFile.read()
imgFile.seek(0)
img1 = jpeg.decode(imgBuf)
img2 = Image.fromarray(np.array(Image.open(imgFile)))
imgFile.close()
f1 = lambda kwargs: lambda _: mm.write(jpeg.encode(img1, **kwargs))
f2 = lambda _: img2.save(mm, 'jpeg')
print('Timing JPEG encoding by libjpeg-turbo: ',
timing(1, f1({'jpeg_subsample': TJSAMP_420}), initial))
print('Timing JPEG encoding by Pillow: ', timing(1, f2, initial))
class RectifierNode(DTROS):
def __init__(self, node_name):
super().__init__(node_name, node_type=NodeType.PERCEPTION)
# parameters
self.publish_freq = DTParam("~publish_freq", -1)
self.alpha = DTParam("~alpha", 0.0)
# utility objects
self.jpeg = TurboJPEG()
self.reminder = DTReminder(frequency=self.publish_freq.value)
self.camera_model = None
self.rect_camera_info = None
self.mapx, self.mapy = None, None
# subscribers
self.sub_img = rospy.Subscriber("~image_in",
CompressedImage,
self.cb_image,
queue_size=1,
buff_size="10MB")
self.sub_camera_info = rospy.Subscriber("~camera_info_in",
CameraInfo,
self.cb_camera_info,
queue_size=1)
# publishers
self.pub_img = rospy.Publisher(
"~image/compressed",
CompressedImage,
queue_size=1,
dt_topic_type=TopicType.PERCEPTION,
dt_healthy_freq=self.publish_freq.value,
dt_help=
"Rectified image (i.e., image with no distortion effects from the lens).",
)
self.pub_camera_info = rospy.Publisher(
"~camera_info",
CameraInfo,
queue_size=1,
dt_topic_type=TopicType.PERCEPTION,
dt_healthy_freq=self.publish_freq.value,
dt_help="Camera parameters for the (virtual) rectified camera.",
)
def cb_camera_info(self, msg):
# unsubscribe from camera_info
self.loginfo(
"Camera info message received. Unsubscribing from camera_info topic."
)
# noinspection PyBroadException
try:
self.sub_camera_info.shutdown()
except BaseException:
pass
# ---
H, W = msg.height, msg.width
# create new camera info
self.camera_model = PinholeCameraModel()
self.camera_model.fromCameraInfo(msg)
# find optimal rectified pinhole camera
with self.profiler("/cb/camera_info/get_optimal_new_camera_matrix"):
rect_camera_K, _ = cv2.getOptimalNewCameraMatrix(
self.camera_model.K, self.camera_model.D, (W, H),
self.alpha.value)
# create rectification map
with self.profiler("/cb/camera_info/init_undistort_rectify_map"):
self.mapx, self.mapy = cv2.initUndistortRectifyMap(
self.camera_model.K, self.camera_model.D, None, rect_camera_K,
(W, H), cv2.CV_32FC1)
# pack rectified camera info into a CameraInfo message
self.rect_camera_info = CameraInfo(
width=W,
height=H,
K=rect_camera_K.flatten().tolist(),
R=np.eye(3).flatten().tolist(),
P=np.zeros((3, 4)).flatten().tolist(),
)
def cb_image(self, msg):
# make sure this matters to somebody
if not self.pub_img.anybody_listening(
) and not self.pub_camera_info.anybody_listening():
return
# make sure we have a map to use
if self.mapx is None or self.mapy is None:
return
# make sure the node is not switched off
if not self.switch:
return
# make sure this is a good time to publish (always keep this as last check)
if not self.reminder.is_time(frequency=self.publish_freq.value):
return
# turn 'compressed distorted image message' into 'raw distorted image'
with self.profiler("/cb/image/decode"):
dist_img = self.jpeg.decode(msg.data)
# run input image through the lens map
with self.profiler("/cb/image/rectify"):
rect_img = cv2.remap(dist_img, self.mapx, self.mapy,
cv2.INTER_NEAREST)
# turn 'raw rectified image' into 'compressed rectified image message'
with self.profiler("/cb/image/encode"):
# rect_img_msg = self.bridge.cv2_to_compressed_imgmsg(rect_img)
rect_img_msg = CompressedImage(format="jpeg",
data=self.jpeg.encode(rect_img))
# maintain original header
rect_img_msg.header.stamp = msg.header.stamp
rect_img_msg.header.frame_id = msg.header.frame_id
self.rect_camera_info.header.stamp = msg.header.stamp
self.rect_camera_info.header.frame_id = msg.header.frame_id
# publish image
self.pub_img.publish(rect_img_msg)
# publish camera info
self.pub_camera_info.publish(self.rect_camera_info)
class TinyDatasetFolder(VisionDataset):
def __init__(self,
root,
loader,
extensions=None,
transform=None,
target_transform=None,
is_valid_file=None,
quality=None):
super(TinyDatasetFolder, self).__init__(root)
self.transform = transform
self.target_transform = target_transform
classes, class_to_idx = self._find_classes(self.root)
samples = make_dataset(self.root, class_to_idx, extensions,
is_valid_file)
if len(samples) == 0:
raise (RuntimeError("Found 0 files in subfolders of: " +
self.root + "\n"
"Supported extensions are: " +
",".join(extensions)))
self.jpeg = TurboJPEG('/home/kai.x/work/local/lib/libturbojpeg.so')
self.loader = loader
self.extensions = extensions
self.classes = classes
self.class_to_idx = class_to_idx
self.samples = samples
self.targets = [s[1] for s in samples]
self.quality = quality
def _find_classes(self, dir):
if sys.version_info >= (3, 5):
# Faster and available in Python 3.5 and above
classes = [d.name for d in os.scandir(dir) if d.is_dir()]
else:
classes = [
d for d in os.listdir(dir)
if os.path.isdir(os.path.join(dir, d))
]
classes.sort()
class_to_idx = {classes[i]: i for i in range(len(classes))}
return classes, class_to_idx
def __getitem__(self, index):
path, target = self.samples[index]
sample = self.loader(path)
# RGB -> BGR
img = np.asarray(sample)
img = img[:, :, ::-1]
# Convert to uint8, this is critical
img = np.ascontiguousarray(img, dtype="uint8")
encoded_img = self.jpeg.encode(img, quality=self.quality)
decoded_img = self.jpeg.decode(encoded_img) # BGR
# BGR -> RGB
sample = decoded_img[:, :, ::-1]
sample = Image.fromarray(sample)
if self.transform is not None:
sample = self.transform(sample)
if self.target_transform is not None:
target = self.target_transform(target)
return sample, target
def __len__(self):
return len(self.samples)
topic = 'processed_video'
consumer = KafkaConsumer(
'video-topic',
bootstrap_servers='localhost:29092',
auto_offset_reset='latest',
enable_auto_commit=True,
group_id='my-group',
)
for message in consumer:
jpeg = TurboJPEG()
bgr_image = jpeg.decode(message.value)
bgr_image = imutils.resize(bgr_image, width=400)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
class SegmentationImageFolder:
def __init__(
self,
root: str,
image_folder: str,
mask_folder: str,
transforms: Optional[Callable] = None,
):
self.image_folder = os.path.join(root, image_folder)
self.mask_folder = os.path.join(root, mask_folder)
self.imgs = list(sorted(os.listdir(self.image_folder)))
self.masks = list(sorted(os.listdir(self.mask_folder)))
self.transforms = transforms
self.jpeg = TurboJPEG(lib_path=local_libturbo_path)
def __getitem__(self, idx: int):
img_path = os.path.join(self.image_folder, self.imgs[idx])
mask_path = os.path.join(self.mask_folder, self.masks[idx])
# img = Image.open(img_path).convert("RGB")
# note that we haven't converted the mask to RGB,
# because each color corresponds to a different instance
# with 0 being background
if img_path.endswith(".jpg") or img_path.endswith(".jpeg"):
fd = open(img_path, 'rb')
img = self.jpeg.decode(fd.read())
fd.close()
else:
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if mask_path.endswith(".jpg") or mask_path.endswith(".jpeg"):
fd = open(mask_path, 'rb')
mask = self.jpeg.decode(fd.read(), pixel_format=TJPF_GRAY)
fd.close()
else:
mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
mask = np.array(mask)
# instances are encoded as different colors
obj_ids = np.unique(mask)
# first id is the background, so remove it
obj_ids = obj_ids[1:]
# split the color-encoded mask into a set
# of binary masks
masks = mask == obj_ids[:, None, None]
# get bounding box coordinates for each mask
num_objs = len(obj_ids)
boxes = []
area = []
for i in range(num_objs):
pos = np.where(masks[i])
xmin = np.min(pos[1])
xmax = np.max(pos[1])
ymin = np.min(pos[0])
ymax = np.max(pos[0])
boxes.append([xmin, ymin, xmax, ymax])
area.append((ymax - ymin) * (xmax - xmin))
target = {}
target["boxes"] = torch.as_tensor(boxes, dtype=torch.float32)
# there is only one class
target["labels"] = torch.ones((num_objs,), dtype=torch.int64)
target["masks"] = torch.as_tensor(masks, dtype=torch.uint8)
target["image_id"] = torch.tensor([idx])
target["area"] = torch.as_tensor(area, dtype=torch.float32)
# suppose all instances are not crowd
target["iscrowd"] = torch.zeros((num_objs,), dtype=torch.int64)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target
def __len__(self):
return len(self.imgs)
class MjpegReader:
def __init__(self, filename):
try:
# Open mjpeg file
self.mjpeg = open(filename, "rb")
# Initialize TurboJpeg (7x faster than opecv embedded jpeg decoder)
self.tjpeg = TurboJPEG()
# File is open
self.opened = True
except FileNotFoundError:
print("File {0} not found".format(filename))
self.opened = False
'''
Read header line
'''
def read_line(self):
ln = ""
# Fine end of line
while True:
# Read each byte and converto to character
character = self.mjpeg.read(1).decode("utf-8")
# Finded end of line, then break loop
if character == '\n':
break
# Concatenate string
ln += character
# Return string
return ln
'''
Read Image
'''
def get_image(self):
# If file is not opened, return None
if not self.opened:
return None
# Read first line (--myboudary) and trash it
self.read_line()
# Read second line X-Timestamp and store timestamp
self.ts = int(self.read_line().split(' ')[1])
# Read content type and remove trash it
self.read_line()
# Read content lenght and get it to read image
self.lenght = int(self.read_line().split(' ')[1])
# Skip blank
self.read_line()
# Return decoded image (numpy image format)
return self.tjpeg.decode(self.mjpeg.read(self.lenght))
'''
Get Timestamp
'''
def get_ts(self):
return self.ts
'''
Get lenght
'''
def get_lenght(self):
return self.lenght
'''
Get file status
'''
def is_opened(self):
return self.opened
def imread(image):
try:
with open(image, 'rb') as _i:
return TurboJPEG.decode(_i.read())
except:
return cv2.imread(image)
time.sleep(1)
turbo_jpeg = TurboJPEG()
cv2.namedWindow("Image")
keep_running = True
idx = 0
t0 = time.time()
while keep_running:
data = grabber.get_buffer()
if data is None:
time.sleep(1)
continue
img = turbo_jpeg.decode(data)
cv2.imshow("Image", img)
keep_running = not (cv2.waitKey(1) & 0xFF == ord('q'))
idx += 1
if idx == 100:
t1 = time.time()
sys.stdout.write("\r {:04} images/second ".format(100 /
(t1 - t0)))
sys.stdout.flush()
t0 = t1
idx = 0
print()
print("Quitting")
grabber.stop()
j_list = []
print('Extractor running (listening to port {})...'.format(iport))
while True:
# get frame info
line = isocket.recv_string()
j = simplejson.loads(line)
dets = j['detections']
imgs = []
# assemble all detected chips into an array
for d in dets:
chip = base64.b64decode(d['chip'])
#fn = 'frame{0}.index{1}.conf{2}.jpg'.format(j['frame'],d['index'],d['conf'])
#with open(fn,'wb') as f:
#f.write(chip)
img = jpeg.decode(chip)
imgs.append(F.to_tensor(np.float32(img)))
# batch-compute embeddings for all chips
aligned = torch.stack(imgs).to(device)
embeddings = resnet(aligned).detach().cpu().numpy()
# store the embeddings in the dictionary we read from the detector
for i, e in enumerate(embeddings):
j['detections'][i]['embedding'] = e.tolist()
j_list.append([nf, j['detections'][i]['chip'], e.tolist()])
#print(j['detections'][i])
#print(e)
#print('\n')
#print('sending [{0}]'.format(j))
# publish send detections-with-embeddings
# osocket.send_string(simplejson.dumps(j)+'\n')
nf += 1