Example #1
0
def process_tfr(path):
    """
    process a waymo tf record into a tf api tf record
    """
    # create processed data dir
    file_name = os.path.basename(path)

    logging.info(f'\nProcessing {path}\n')

    writer = tf.python_io.TFRecordWriter(f'output/{file_name}')

    dataset = tf.data.TFRecordDataset(path, compression_type='')
    for idx, data in enumerate(dataset):
        frame = open_dataset.Frame()
        frame.ParseFromString(bytearray(data.numpy()))
        encoded_jpeg, annotations = parse_frame(frame)

        filename = file_name.replace('.tfrecord', f'_{idx}.tfrecord')
        tf_example = create_tf_example(filename, encoded_jpeg, annotations)

        writer.write(tf_example.SerializeToString())

    writer.close()

    logging.info(f'\nFinish {path}\n')
def process_tfr(path, data_dir):
    """
    process a Waymo tf record into a tf api tf record

    args:
        - path [str]: path to the Waymo tf record file
        - data_dir [str]: path to the destination directory
    """
    # create processed data dir
    dest = os.path.join(data_dir, 'processed')
    os.makedirs(dest, exist_ok=True)
    file_name = os.path.basename(path)

    logger.info(f'Processing {path}')
    writer = tf.python_io.TFRecordWriter(f'{dest}/{file_name}')
    dataset = tf.data.TFRecordDataset(path, compression_type='')
    for idx, data in enumerate(dataset):
        # we are only saving every 10 frames to reduce the number of similar
        # images. Remove this line if you have enough space to work with full
        # temporal resolution data.
        if idx % 10 == 0:
            frame = open_dataset.Frame()
            frame.ParseFromString(bytearray(data.numpy()))
            encoded_jpeg, annotations = parse_frame(frame)
            filename = file_name.replace('.tfrecord', f'_{idx}.tfrecord')
            tf_example = create_tf_example(filename, encoded_jpeg, annotations)
            writer.write(tf_example.SerializeToString())
    writer.close()
Example #3
0
    def __receiver_loop(self):
        """
        Main loop listening for incoming data.
        """
        try:
            try:
                threading.currentThread().setName("StompReceiver")
                while self.__running:
                    log.debug('starting receiver loop')

                    if self.__socket is None:
                        break

                    try:
                        try:
                            self.__notify('connecting')
                            
                            while self.__running:
                                frames = self.__read()
                                
                                for frame in frames:
                                    (frame_type, headers, body) = utils.parse_frame(frame)
                                    log.debug("Received frame: result=%r, headers=%r, body=%r" % (frame_type, headers, body))
                                    frame_type = frame_type.lower()
                                    if frame_type in [ 'connected', 'message', 'receipt', 'error' ]:
                                        self.__notify(frame_type, headers, body)
                                    else:
                                        log.warning('Unknown response frame type: "%s" (frame length was %d)' % (frame_type, len(frame)))
                        finally:
                            try:
                                self.__socket.close()
                            except:
                                pass # ignore errors when attempting to close socket
                            self.__socket = None
                            self.__current_host_and_port = None
                    except exception.ConnectionClosedException:
                        if self.__running:
                            log.error("Lost connection")
                            self.__notify('disconnected')
                            #
                            # Clear out any half-received messages after losing connection
                            #
                            self.__recvbuf = ''
                            continue
                        else:
                            break
            except:
                log.exception("An unhandled exception was encountered in the stomp receiver loop")

        finally:
            self.__receiver_thread_exit_condition.acquire()
            self.__receiver_thread_exited = True
            self.__receiver_thread_exit_condition.notifyAll()
            self.__receiver_thread_exit_condition.release()
Example #4
0
    def __receiver_loop(self):
        """
        Main loop listening for incoming data.
        """
        log.debug("Starting receiver loop")
        try:
            try:
                while self.__running:
                    if self.__socket is None:
                        break

                    try:
                        try:
                            while self.__running:
                                frames = self.__read()
                                
                                for frame in frames:
                                    (frame_type, headers, body) = utils.parse_frame(frame)
                                    log.debug("Received frame: %r, headers=%r, body=%r" % (frame_type, headers, body))
                                    frame_type = frame_type.lower()
                                    if frame_type in [ 'connected', 'message', 'receipt', 'error' ]:
                                        self.__notify(frame_type, headers, body)
                                    elif frame_type == 'heartbeat':
                                        # no notifications needed
                                        pass
                                    else:
                                        log.warning('Unknown response frame type: "%s" (frame length was %d)' % (frame_type, len(frame)))
                        finally:
                            try:
                                self.__socket.close()
                            except:
                                pass # ignore errors when attempting to close socket
                            self.__socket = None
                            self.__current_host_and_port = None
                    except exception.ConnectionClosedException:
                        if self.__running:
                            log.error("Lost connection")
                            self.__notify('disconnected')
                            #
                            # Clear out any half-received messages after losing connection
                            #
                            self.__recvbuf = ''
                            self.__running = False
                        break
            except:
                log.exception("An unhandled exception was encountered in the stomp receiver loop")

        finally:
            self.__receiver_thread_exit_condition.acquire()
            self.__receiver_thread_exited = True
            self.__receiver_thread_exit_condition.notifyAll()
            self.__receiver_thread_exit_condition.release()
            log.debug("Receiver loop ended")
Example #5
0
    def __receiver_loop(self):
        """
        Main loop listening for incoming data.
        """
        log.debug("Starting receiver loop")
        try:
            try:
                while self.running:
                    if self.socket is None:
                        break

                    try:
                        try:
                            while self.running:
                                frames = self.__read()
                                
                                for frame in frames:
                                    f = utils.parse_frame(frame)
                                    self.process_frame(f, frame)
                        except:
                            _, e, _ = sys.exc_info()
                            print(e)
                        finally:
                            try:
                                self.socket.close()
                            except:
                                pass # ignore errors when attempting to close socket
                            self.socket = None
                            self.current_host_and_port = None
                    except exception.ConnectionClosedException:
                        if self.running:
                            self.notify('disconnected')
                            #
                            # Clear out any half-received messages after losing connection
                            #
                            self.__recvbuf = ''
                            self.running = False
                        break
            except:
                log.exception("An unhandled exception was encountered in the stomp receiver loop")

        finally:
            self.__receiver_thread_exit_condition.acquire()
            self.__receiver_thread_exited = True
            self.__receiver_thread_exit_condition.notifyAll()
            self.__receiver_thread_exit_condition.release()
            log.debug("Receiver loop ended")
Example #6
0
    def __receiver_loop(self):
        """
        Main loop listening for incoming data.
        """
        log.info("Starting receiver loop")
        try:
            while self.running:
                if self.socket is None:
                    break

                try:
                    while self.running:
                        frames = self.__read()

                        for frame in frames:
                            f = utils.parse_frame(frame)
                            self.process_frame(f, frame)
                except exception.ConnectionClosedException:
                    if self.running:
                        self.notify('disconnected')
                        #
                        # Clear out any half-received messages after losing connection
                        #
                        self.__recvbuf = ''
                        self.running = False
                    break
                finally:
                    try:
                        self.socket.close()
                    except:
                        pass  # ignore errors when attempting to close socket
                    self.socket = None
                    self.current_host_and_port = None
        finally:
            self.__receiver_thread_exit_condition.acquire()
            self.__receiver_thread_exited = True
            self.__receiver_thread_exit_condition.notifyAll()
            self.__receiver_thread_exit_condition.release()
            log.info("Receiver loop ended")
Example #7
0
    def __receiver_loop(self):
        """
        Main loop listening for incoming data.
        """
        log.debug("Starting receiver loop")
        try:
            while self.running:
                if self.socket is None:
                    break

                try:
                    while self.running:
                        frame = self.__read().decode('utf-8')
                        f = utils.parse_frame(frame)
                        self.process_frame(f, frame)
                except _exceptions.WebSocketConnectionClosedException:
                    if self.running:
                        self.running = False
                except IndexError:
                    logging.exception("Something bad happened - Disconnecting!")
                    if self.running:
                        self.running = False
                finally:
                    try:
                        if self.socket:
                            self.socket.close()
                    except:
                        pass  # ignore errors when attempting to close socket
                    self.notify('disconnected')
                    self.socket = None
                    self.current_host_and_port = None
        finally:
            self.__receiver_thread_exit_condition.acquire()
            self.__receiver_thread_exited = True
            self.__receiver_thread_exit_condition.notifyAll()
            self.__receiver_thread_exit_condition.release()
            log.debug("Receiver loop ended")