def Proc(record): """Parses a serialized tf.Example record.""" features = [ ('uttid', tf.VarLenFeature(tf.string)), ('transcript', tf.VarLenFeature(tf.string)), ('frames', tf.VarLenFeature(tf.float32)), ] example = tf.parse_single_example(record, dict(features)) fval = {k: v.values for k, v in six.iteritems(example)} # Reshape the flattened vector into its original time-major # representation. fval['frames'] = tf.reshape(fval['frames'], shape=[-1, self.params.frame_size]) # Input duration determines the bucket. bucket_key = tf.cast(tf.shape(fval['frames'])[0], tf.int32) if self.params.append_eos_frame: bucket_key += 1 tgt_ids, tgt_labels, tgt_paddings = self.StringsToIds( fval['transcript']) src_paddings = tf.zeros([tf.shape(fval['frames'])[0]], dtype=tf.float32) return [ fval['uttid'], tgt_ids, tgt_labels, tgt_paddings, fval['frames'], src_paddings ], bucket_key
def Proc(record): """Parses a serialized tf.Example record.""" outputs = [ ('inputs', tf.VarLenFeature(tf.int64)), ('targets', tf.VarLenFeature(tf.int64)), ] features = tf.parse_single_example(record, dict(outputs)) for k, v in six.iteritems(features): features[k] = v.values src_ids = features['inputs'] tgt_labels = features['targets'] # Derive src_paddings, tgt_ids, tgt_paddings. # tgt_ids is tgt_labels shifted right by one, with a SOS ID prepended. tgt_ids = tf.concat([[p.sos_id], tgt_labels[:-1]], axis=0) src_paddings = tf.zeros(tf.shape(src_ids), dtype=tf.float32) tgt_paddings = tf.zeros(tf.shape(tgt_ids), dtype=tf.float32) tgt_weights = tf.ones(tf.shape(tgt_ids), dtype=tf.float32) bucket_key = tf.cast( tf.maximum( tf.reduce_sum(1.0 - src_paddings), tf.reduce_sum(1.0 - tgt_paddings)), tf.int32) return [ src_ids, src_paddings, tgt_ids, tgt_paddings, tgt_labels, tgt_weights ], bucket_key
def FeatureMap(self): """Return a dictionary from tf.Example feature names to Features.""" p = self.params feature_map = {} for laser in p.cbr_laser_names + p.gbr_laser_names: feature_map['%s_beam_inclinations' % laser] = (tf.VarLenFeature(dtype=tf.float32)) feature_map['%s_beam_inclination_min' % laser] = (tf.VarLenFeature(dtype=tf.float32)) feature_map['%s_beam_inclination_max' % laser] = (tf.VarLenFeature(dtype=tf.float32)) feature_map['%s_extrinsics' % laser] = tf.VarLenFeature(dtype=tf.float32) if laser in p.gbr_laser_names: feature_map['%s_pose' % laser] = tf.VarLenFeature(dtype=tf.float32) for returns in p.returns: feature_map['%s_%s' % (laser, returns)] = tf.VarLenFeature( dtype=tf.float32) feature_map['%s_%s_shape' % (laser, returns)] = tf.VarLenFeature( dtype=tf.int64) feature_map['pose'] = tf.VarLenFeature(dtype=tf.float32) return feature_map
def FeatureMap(self): """Return a dictionary from tf.Example feature names to Features.""" p = self.params features = {} features['pose'] = tf.VarLenFeature(dtype=tf.float32) for camera_name in p.camera_names: features['image_%s' % camera_name] = tf.VarLenFeature(dtype=tf.string) features['image_%s_shape' % camera_name] = ( tf.VarLenFeature(dtype=tf.int64)) features['camera_%s_intrinsics' % camera_name] = tf.VarLenFeature(dtype=tf.float32) features['camera_%s_extrinsics' % camera_name] = tf.VarLenFeature(dtype=tf.float32) features['camera_%s_rolling_shutter_direction' % camera_name] = tf.FixedLenFeature( dtype=tf.int64, shape=()) features['image_%s_pose' % camera_name] = tf.VarLenFeature(dtype=tf.float32) features['image_%s_velocity' % camera_name] = tf.VarLenFeature(dtype=tf.float32) for feat in [ 'pose_timestamp', 'shutter', 'camera_trigger_time', 'camera_readout_done_time' ]: features['image_%s_%s' % (camera_name, feat)] = tf.FixedLenFeature( dtype=tf.float32, shape=()) return features
def FeatureMap(self): """Return a dictionary from tf.Example feature names to Features.""" p = self.params features = {} for lidar in p.lidar_names: for ri in ['ri1', 'ri2']: features['laser_%s_%s' % (lidar, ri)] = tf.VarLenFeature(dtype=tf.float32) return features
def FeatureMap(self): """Return a dictionary from tf.Example feature names to Features.""" feature_map = {} feature_map['pose'] = tf.VarLenFeature(dtype=tf.float32) feature_map['run_segment'] = tf.FixedLenFeature((), tf.string, '') feature_map['run_start_offset'] = tf.FixedLenFeature((), tf.int64, 0) feature_map['time_of_day'] = tf.FixedLenFeature((), tf.string, '') feature_map['location'] = tf.FixedLenFeature((), tf.string, '') feature_map['weather'] = tf.FixedLenFeature((), tf.string, '') return feature_map
def _Proc(record): """Parses a serialized tf.Example record.""" outputs = [ ('inputs', tf.VarLenFeature(tf.int64)), ('targets', tf.VarLenFeature(tf.int64)), ] features = tf.parse_single_example(record, dict(outputs)) for k, v in six.iteritems(features): features[k] = v.values src_ids = features['inputs'] tgt_labels = features['targets'] src_paddings, tgt_ids, tgt_paddings, tgt_weights, bucket_key = _DerivePaddingsAndIds( src_ids, tgt_labels) return [ src_ids, src_paddings, tgt_ids, tgt_paddings, tgt_labels, tgt_weights ], bucket_key
def FeatureMap(self): """Return a dictionary from tf.Example feature names to Features.""" feature_map = {} feature_map['labels'] = tf.VarLenFeature(dtype=tf.int64) feature_map['label_ids'] = tf.VarLenFeature(dtype=tf.string) feature_map['detection_difficulties'] = tf.VarLenFeature(dtype=tf.int64) feature_map['combined_detection_difficulties'] = tf.VarLenFeature( dtype=tf.int64) feature_map['tracking_difficulties'] = tf.VarLenFeature(dtype=tf.int64) feature_map['bboxes_3d'] = tf.VarLenFeature(dtype=tf.float32) feature_map['bboxes_3d_num_points'] = tf.VarLenFeature(dtype=tf.int64) feature_map['label_metadata'] = tf.VarLenFeature(dtype=tf.float32) return feature_map
def _Proc(record): """Parses a serialized tf.Example record.""" outputs = [ ('inputs', tf.VarLenFeature(tf.int64)), ('targets', tf.VarLenFeature(tf.int64)), ] features = tf.parse_single_example(record, dict(outputs)) for k, v in six.iteritems(features): features[k] = v.values src_ids = features['inputs'] tgt_labels = features['targets'] # Derive trivial segmentation for unpacked input. src_paddings, tgt_ids, tgt_paddings, tgt_weights, bucket_key = _DerivePaddingsAndIds( src_ids, tgt_labels) src_len = tf.shape(src_ids)[0] tgt_len = tf.shape(tgt_ids)[0] src_pos = tf.range(src_len, dtype=tf.int32) src_seg = tf.zeros_like(src_paddings) tgt_pos = tf.range(tgt_len, dtype=tf.int32) tgt_seg = tf.zeros_like(tgt_paddings) return [ src_ids, src_paddings, tgt_ids, tgt_paddings, tgt_labels, tgt_weights, src_pos, src_seg, tgt_pos, tgt_seg, ], bucket_key
def Proc(record): """Parses a serialized tf.Example record.""" outputs = [ ('source_id', tf.VarLenFeature(tf.int64)), ('source_padding', tf.VarLenFeature(tf.float32)), ('target_id', tf.VarLenFeature(tf.int64)), ('target_padding', tf.VarLenFeature(tf.float32)), ('target_label', tf.VarLenFeature(tf.int64)), ('target_weight', tf.VarLenFeature(tf.float32)), ] features = tf.parse_single_example(record, dict(outputs)) for k, v in six.iteritems(features): features[k] = v.values bucket_key = tf.to_int32( tf.maximum(tf.reduce_sum(1.0 - features['source_padding']), tf.reduce_sum(1.0 - features['target_padding']))) return [features[k] for k, _ in outputs] + [bucket_key]
def _ProcPacked(record): """TFExample -> Tensors for PackedInput.""" outputs = [ ('inputs', tf.VarLenFeature(tf.int64)), ('targets', tf.VarLenFeature(tf.int64)), ('inputs_segmentation', tf.VarLenFeature(tf.int64)), ('inputs_position', tf.VarLenFeature(tf.int64)), ('targets_segmentation', tf.VarLenFeature(tf.int64)), ('targets_position', tf.VarLenFeature(tf.int64)), ] features = tf.parse_single_example(record, dict(outputs)) for k, v in six.iteritems(features): features[k] = v.values src_ids = features['inputs'] tgt_labels = features['targets'] src_pos = features['inputs_position'] src_seg = features['inputs_segmentation'] tgt_pos = features['targets_position'] tgt_seg = features['targets_segmentation'] src_paddings, tgt_ids, tgt_paddings, tgt_weights, bucket_key = _DerivePaddingsAndIds( src_ids, tgt_labels) return [ src_ids, src_paddings, tgt_ids, tgt_paddings, tgt_labels, tgt_weights, src_pos, src_seg, tgt_pos, tgt_seg, ], bucket_key
def FeatureMap(self): feature_map = { 'pointcloud/xyz': tf.VarLenFeature(dtype=tf.float32), 'pointcloud/reflectance': tf.VarLenFeature(dtype=tf.float32), } return feature_map
def FeatureMap(self): return { 'image/source_id': tf.FixedLenFeature((), tf.string, ''), 'object/image/bbox/xmin': tf.VarLenFeature(tf.float32), 'object/image/bbox/xmax': tf.VarLenFeature(tf.float32), 'object/image/bbox/ymin': tf.VarLenFeature(tf.float32), 'object/image/bbox/ymax': tf.VarLenFeature(tf.float32), 'object/label': tf.VarLenFeature(tf.string), 'object/has_3d_info': tf.VarLenFeature(dtype=tf.int64), 'object/occlusion': tf.VarLenFeature(dtype=tf.int64), 'object/truncation': tf.VarLenFeature(dtype=tf.float32), 'object/velo/bbox/xyz': tf.VarLenFeature(dtype=tf.float32), 'object/velo/bbox/dim_xyz': tf.VarLenFeature(dtype=tf.float32), 'object/velo/bbox/phi': tf.VarLenFeature(dtype=tf.float32), 'transform/velo_to_image_plane': tf.FixedLenFeature(shape=(3, 4), dtype=tf.float32), }