def process_tfr(path):
"""
process a waymo tf record into a tf api tf record
"""
# create processed data dir
file_name = os.path.basename(path)
logging.info(f'\nProcessing {path}\n')
writer = tf.python_io.TFRecordWriter(f'output/{file_name}')
dataset = tf.data.TFRecordDataset(path, compression_type='')
for idx, data in enumerate(dataset):
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
encoded_jpeg, annotations = parse_frame(frame)
filename = file_name.replace('.tfrecord', f'_{idx}.tfrecord')
tf_example = create_tf_example(filename, encoded_jpeg, annotations)
writer.write(tf_example.SerializeToString())
writer.close()
logging.info(f'\nFinish {path}\n')
def process_tfr(path, data_dir):
"""
process a Waymo tf record into a tf api tf record
args:
- path [str]: path to the Waymo tf record file
- data_dir [str]: path to the destination directory
"""
# create processed data dir
dest = os.path.join(data_dir, 'processed')
os.makedirs(dest, exist_ok=True)
file_name = os.path.basename(path)
logger.info(f'Processing {path}')
writer = tf.python_io.TFRecordWriter(f'{dest}/{file_name}')
dataset = tf.data.TFRecordDataset(path, compression_type='')
for idx, data in enumerate(dataset):
# we are only saving every 10 frames to reduce the number of similar
# images. Remove this line if you have enough space to work with full
# temporal resolution data.
if idx % 10 == 0:
frame = open_dataset.Frame()
frame.ParseFromString(bytearray(data.numpy()))
encoded_jpeg, annotations = parse_frame(frame)
filename = file_name.replace('.tfrecord', f'_{idx}.tfrecord')
tf_example = create_tf_example(filename, encoded_jpeg, annotations)
writer.write(tf_example.SerializeToString())
writer.close()
def __receiver_loop(self):
"""
Main loop listening for incoming data.
"""
try:
try:
threading.currentThread().setName("StompReceiver")
while self.__running:
log.debug('starting receiver loop')
if self.__socket is None:
break
try:
try:
self.__notify('connecting')
while self.__running:
frames = self.__read()
for frame in frames:
(frame_type, headers, body) = utils.parse_frame(frame)
log.debug("Received frame: result=%r, headers=%r, body=%r" % (frame_type, headers, body))
frame_type = frame_type.lower()
if frame_type in [ 'connected', 'message', 'receipt', 'error' ]:
self.__notify(frame_type, headers, body)
else:
log.warning('Unknown response frame type: "%s" (frame length was %d)' % (frame_type, len(frame)))
finally:
try:
self.__socket.close()
except:
pass # ignore errors when attempting to close socket
self.__socket = None
self.__current_host_and_port = None
except exception.ConnectionClosedException:
if self.__running:
log.error("Lost connection")
self.__notify('disconnected')
#
# Clear out any half-received messages after losing connection
#
self.__recvbuf = ''
continue
else:
break
except:
log.exception("An unhandled exception was encountered in the stomp receiver loop")
finally:
self.__receiver_thread_exit_condition.acquire()
self.__receiver_thread_exited = True
self.__receiver_thread_exit_condition.notifyAll()
self.__receiver_thread_exit_condition.release()
def __receiver_loop(self):
"""
Main loop listening for incoming data.
"""
log.debug("Starting receiver loop")
try:
try:
while self.__running:
if self.__socket is None:
break
try:
try:
while self.__running:
frames = self.__read()
for frame in frames:
(frame_type, headers, body) = utils.parse_frame(frame)
log.debug("Received frame: %r, headers=%r, body=%r" % (frame_type, headers, body))
frame_type = frame_type.lower()
if frame_type in [ 'connected', 'message', 'receipt', 'error' ]:
self.__notify(frame_type, headers, body)
elif frame_type == 'heartbeat':
# no notifications needed
pass
else:
log.warning('Unknown response frame type: "%s" (frame length was %d)' % (frame_type, len(frame)))
finally:
try:
self.__socket.close()
except:
pass # ignore errors when attempting to close socket
self.__socket = None
self.__current_host_and_port = None
except exception.ConnectionClosedException:
if self.__running:
log.error("Lost connection")
self.__notify('disconnected')
#
# Clear out any half-received messages after losing connection
#
self.__recvbuf = ''
self.__running = False
break
except:
log.exception("An unhandled exception was encountered in the stomp receiver loop")
finally:
self.__receiver_thread_exit_condition.acquire()
self.__receiver_thread_exited = True
self.__receiver_thread_exit_condition.notifyAll()
self.__receiver_thread_exit_condition.release()
log.debug("Receiver loop ended")
def __receiver_loop(self):
"""
Main loop listening for incoming data.
"""
log.debug("Starting receiver loop")
try:
try:
while self.running:
if self.socket is None:
break
try:
try:
while self.running:
frames = self.__read()
for frame in frames:
f = utils.parse_frame(frame)
self.process_frame(f, frame)
except:
_, e, _ = sys.exc_info()
print(e)
finally:
try:
self.socket.close()
except:
pass # ignore errors when attempting to close socket
self.socket = None
self.current_host_and_port = None
except exception.ConnectionClosedException:
if self.running:
self.notify('disconnected')
#
# Clear out any half-received messages after losing connection
#
self.__recvbuf = ''
self.running = False
break
except:
log.exception("An unhandled exception was encountered in the stomp receiver loop")
finally:
self.__receiver_thread_exit_condition.acquire()
self.__receiver_thread_exited = True
self.__receiver_thread_exit_condition.notifyAll()
self.__receiver_thread_exit_condition.release()
log.debug("Receiver loop ended")
def __receiver_loop(self):
"""
Main loop listening for incoming data.
"""
log.info("Starting receiver loop")
try:
while self.running:
if self.socket is None:
break
try:
while self.running:
frames = self.__read()
for frame in frames:
f = utils.parse_frame(frame)
self.process_frame(f, frame)
except exception.ConnectionClosedException:
if self.running:
self.notify('disconnected')
#
# Clear out any half-received messages after losing connection
#
self.__recvbuf = ''
self.running = False
break
finally:
try:
self.socket.close()
except:
pass # ignore errors when attempting to close socket
self.socket = None
self.current_host_and_port = None
finally:
self.__receiver_thread_exit_condition.acquire()
self.__receiver_thread_exited = True
self.__receiver_thread_exit_condition.notifyAll()
self.__receiver_thread_exit_condition.release()
log.info("Receiver loop ended")
def __receiver_loop(self):
"""
Main loop listening for incoming data.
"""
log.debug("Starting receiver loop")
try:
while self.running:
if self.socket is None:
break
try:
while self.running:
frame = self.__read().decode('utf-8')
f = utils.parse_frame(frame)
self.process_frame(f, frame)
except _exceptions.WebSocketConnectionClosedException:
if self.running:
self.running = False
except IndexError:
logging.exception("Something bad happened - Disconnecting!")
if self.running:
self.running = False
finally:
try:
if self.socket:
self.socket.close()
except:
pass # ignore errors when attempting to close socket
self.notify('disconnected')
self.socket = None
self.current_host_and_port = None
finally:
self.__receiver_thread_exit_condition.acquire()
self.__receiver_thread_exited = True
self.__receiver_thread_exit_condition.notifyAll()
self.__receiver_thread_exit_condition.release()
log.debug("Receiver loop ended")