def _test_build_transforming_training_input_fn(self, shape): basedir = tempfile.mkdtemp() raw_metadata = dataset_metadata.DatasetMetadata( schema=_make_raw_schema(shape)) # the transformed schema should be vectorized already. transformed_metadata = dataset_metadata.DatasetMetadata( schema=_make_transformed_schema([1])) data_file = os.path.join(basedir, 'data') examples = [ _create_serialized_example(d) for d in [{ 'raw_a': 15, 'raw_b': 6, 'raw_label': 77 }, { 'raw_a': 12, 'raw_b': 17, 'raw_label': 44 }] ] _write_tfrecord(data_file, examples) transform_savedmodel_dir = os.path.join(basedir, 'transform-savedmodel') _write_transform_savedmodel(transform_savedmodel_dir) training_input_fn = ( input_fn_maker.build_transforming_training_input_fn( raw_metadata=raw_metadata, transformed_metadata=transformed_metadata, transform_savedmodel_dir=transform_savedmodel_dir, raw_data_file_pattern=[data_file], training_batch_size=128, raw_label_keys=['raw_label'], transformed_label_keys=['transformed_label'], raw_feature_keys=['raw_a', 'raw_b'], transformed_feature_keys=['transformed_a', 'transformed_b'], randomize_input=False)) with tf.Graph().as_default(): features, labels = training_input_fn() with tf.Session().as_default() as session: session.run(tf.initialize_all_variables()) tf.train.start_queue_runners() transformed_a, transformed_b, transformed_label = session.run([ features['transformed_a'], features['transformed_b'], labels ]) self.assertEqual(21, transformed_a[0][0]) self.assertEqual(9, transformed_b[0][0]) self.assertEqual(77000, transformed_label[0][0]) self.assertEqual(29, transformed_a[1][0]) self.assertEqual(-5, transformed_b[1][0]) self.assertEqual(44000, transformed_label[1][0])
def _test_build_transforming_training_input_fn(self, shape): basedir = tempfile.mkdtemp() raw_metadata = dataset_metadata.DatasetMetadata( schema=_make_raw_schema(shape, should_add_unused_feature=True)) # the transformed schema should be vectorized already. transformed_metadata = dataset_metadata.DatasetMetadata( schema=_make_transformed_schema([1])) data_file = os.path.join(basedir, 'data') examples = [_create_serialized_example(d) for d in [ {'raw_a': 15, 'raw_b': 6, 'raw_label': 77}, {'raw_a': 12, 'raw_b': 17, 'raw_label': 44}]] _write_tfrecord(data_file, examples) transform_savedmodel_dir = os.path.join(basedir, 'transform-savedmodel') _write_transform_savedmodel( transform_savedmodel_dir, should_add_unused_feature=True) training_input_fn = ( input_fn_maker.build_transforming_training_input_fn( raw_metadata=raw_metadata, transformed_metadata=transformed_metadata, transform_savedmodel_dir=transform_savedmodel_dir, raw_data_file_pattern=[data_file], training_batch_size=128, transformed_label_keys=['transformed_label'], randomize_input=False, convert_scalars_to_vectors=True)) with tf.Graph().as_default(): features, labels = training_input_fn() with tf.Session().as_default() as session: session.run(tf.initialize_all_variables()) tf.train.start_queue_runners() transformed_a, transformed_b, transformed_label = session.run( [features['transformed_a'], features['transformed_b'], labels]) batch_shape = (128, 1) sparse_batch_shape = batch_shape if not shape: # transformed_b is sparse so _convert_scalars_to_vectors did not fix it sparse_batch_shape = sparse_batch_shape[:1] transformed_b_dict = dict(zip([tuple(x + [0]) for x in transformed_b.indices.tolist()], transformed_b.values.tolist())) else: transformed_b_dict = dict(zip([tuple(x) for x in transformed_b.indices.tolist()], transformed_b.values.tolist())) self.assertEqual(batch_shape, tuple(transformed_a.shape)) self.assertEqual(sparse_batch_shape, tuple(transformed_b.dense_shape)) self.assertEqual(batch_shape, tuple(transformed_label.shape)) self.assertEqual(21, transformed_a[0][0]) self.assertEqual(9, transformed_b_dict[(0, 0)]) self.assertEqual(77000, transformed_label[0][0]) self.assertEqual(29, transformed_a[1][0]) self.assertEqual(-5, transformed_b_dict[(1, 0)]) self.assertEqual(44000, transformed_label[1][0])