def testSentimentExampleAccuracy(self):
raw_data_dir = os.path.join(os.path.dirname(__file__),
'testdata/sentiment')
working_dir = self.get_temp_dir()
# Copy data from raw data directory to `working_dir`
for filename in [
'test_shuffled-00000-of-00001', 'train_shuffled-00000-of-00001'
]:
shutil.copy(os.path.join(raw_data_dir, filename), working_dir)
sentiment_example.transform_data(working_dir)
results = sentiment_example.train_and_evaluate(
working_dir, num_train_instances=1000, num_test_instances=1000)
self.assertGreaterEqual(results['accuracy'], 0.7)
# Delete temp directory and transform_fn directory. This ensures that the
# test of serving the model below will only pass if the SavedModel saved
# to sentiment_example.EXPORTED_MODEL_DIR is hermetic, i.e does not contain
# references to tft_temp and transform_fn.
shutil.rmtree(
os.path.join(working_dir, sentiment_example.TRANSFORM_TEMP_DIR))
shutil.rmtree(
os.path.join(working_dir, tft.TFTransformOutput.TRANSFORM_FN_DIR))
if local_model_server.local_model_server_supported():
model_name = 'my_model'
model_path = os.path.join(working_dir,
sentiment_example.EXPORTED_MODEL_DIR)
with local_model_server.start_server(model_name,
model_path) as address:
# Use made up data chosen to give high probability of negative
# sentiment.
ascii_classification_request = """model_spec { name: "my_model" }
input {
example_list {
examples {
features {
feature {
key: "review"
value: {
bytes_list {
value: "errible terrible terrible terrible terrible terrible terrible."
}
}
}
}
}
}
}"""
results = local_model_server.make_classification_request(
address, ascii_classification_request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].classes), 2)
self.assertEqual(results[0].classes[0].label, '0')
self.assertGreater(results[0].classes[0].score, 0.8)
self.assertEqual(results[0].classes[1].label, '1')
self.assertLess(results[0].classes[1].score, 0.2)
def testSentimentExampleAccuracy(self):
raw_data_dir = os.path.join(os.path.dirname(__file__),
'testdata/sentiment')
working_dir = self.get_temp_dir()
# Copy data from raw data directory to `working_dir`
for filename in [
'test_shuffled-00000-of-00001', 'train_shuffled-00000-of-00001'
]:
shutil.copy(os.path.join(raw_data_dir, filename), working_dir)
sentiment_example.transform_data(working_dir)
results = sentiment_example.train_and_evaluate(
working_dir, num_train_instances=1000, num_test_instances=1000)
self.assertGreaterEqual(results['accuracy'], 0.7)
if local_model_server.local_model_server_supported():
model_name = 'my_model'
model_path = os.path.join(working_dir,
sentiment_example.EXPORTED_MODEL_DIR)
with local_model_server.start_server(model_name,
model_path) as address:
# Use made up data chosen to give high probability of negative
# sentiment.
ascii_classification_request = """model_spec { name: "my_model" }
input {
example_list {
examples {
features {
feature {
key: "review"
value: {
bytes_list {
value: "errible terrible terrible terrible terrible terrible terrible."
}
}
}
}
}
}
}"""
results = local_model_server.make_classification_request(
address, ascii_classification_request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].classes), 2)
self.assertEqual(results[0].classes[0].label, '0')
self.assertGreater(results[0].classes[0].score, 0.8)
self.assertEqual(results[0].classes[1].label, '1')
self.assertLess(results[0].classes[1].score, 0.2)
def testCensusExampleAccuracy(self, read_raw_data_for_training):
if not self._should_saved_model_load_work():
self.skipTest(
'The generated SavedModel cannot be read with TF<2.2')
raw_data_dir = self._get_data_dir()
working_dir = self._get_working_dir()
train_data_file = os.path.join(raw_data_dir, 'adult.data')
test_data_file = os.path.join(raw_data_dir, 'adult.test')
census_example_common.transform_data(train_data_file, test_data_file,
working_dir)
if read_raw_data_for_training:
raw_train_and_eval_patterns = (train_data_file, test_data_file)
transformed_train_and_eval_patterns = None
else:
train_pattern = os.path.join(
working_dir,
census_example_common.TRANSFORMED_TRAIN_DATA_FILEBASE + '*')
eval_pattern = os.path.join(
working_dir,
census_example_common.TRANSFORMED_TEST_DATA_FILEBASE + '*')
raw_train_and_eval_patterns = None
transformed_train_and_eval_patterns = (train_pattern, eval_pattern)
output_dir = os.path.join(working_dir,
census_example_common.EXPORTED_MODEL_DIR)
results = census_example_v2.train_and_evaluate(
raw_train_and_eval_patterns,
transformed_train_and_eval_patterns,
output_dir,
working_dir,
num_train_instances=1000,
num_test_instances=1000)
self.assertGreaterEqual(results[1], 0.7)
# Removing the tf.Transform output directory in order to show that the
# exported model is hermetic.
shutil.rmtree(os.path.join(working_dir, 'transform_fn'))
model_path = os.path.join(working_dir,
census_example_common.EXPORTED_MODEL_DIR)
actual_model_path = os.path.join(model_path, '1')
tf.keras.backend.clear_session()
model = tf.keras.models.load_model(actual_model_path)
model.summary()
example = text_format.Parse(_PREDICT_TF_EXAMPLE_TEXT_PB,
tf.train.Example())
prediction = model.signatures['serving_default'](tf.constant(
[example.SerializeToString()], tf.string))
self.assertAllEqual([['0', '1']], prediction['classes'])
self.assertAllClose([[0, 1]], prediction['scores'], atol=0.001)
# This is required in order to support the classify API for this Keras
# model.
updater = tf.compat.v1.saved_model.signature_def_utils.MethodNameUpdater(
actual_model_path)
updater.replace_method_name(signature_key='serving_default',
method_name='tensorflow/serving/classify',
tags=['serve'])
updater.save()
if local_model_server.local_model_server_supported():
with local_model_server.start_server(_MODEL_NAME,
model_path) as address:
ascii_classification_request = _CLASSIFICATION_REQUEST_TEXT_PB
results = local_model_server.make_classification_request(
address, ascii_classification_request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].classes), 2)
self.assertEqual(results[0].classes[0].label, '0')
self.assertLess(results[0].classes[0].score, 0.01)
self.assertEqual(results[0].classes[1].label, '1')
self.assertGreater(results[0].classes[1].score, 0.99)
def testCensusExampleAccuracy(self):
raw_data_dir = os.path.join(os.path.dirname(__file__), 'testdata/census')
working_dir = self.get_temp_dir()
train_data_file = os.path.join(raw_data_dir, 'adult.data')
test_data_file = os.path.join(raw_data_dir, 'adult.test')
census_example_common.transform_data(train_data_file, test_data_file,
working_dir)
results = census_example.train_and_evaluate(
working_dir, num_train_instances=1000, num_test_instances=1000)
self.assertGreaterEqual(results['accuracy'], 0.7)
if local_model_server.local_model_server_supported():
model_name = 'my_model'
model_path = os.path.join(working_dir,
census_example_common.EXPORTED_MODEL_DIR)
with local_model_server.start_server(model_name, model_path) as address:
# Use first row of test data set, which has high probability on label 1
# (which corresponds to '<=50K').
ascii_classification_request = """model_spec { name: "my_model" }
input {
example_list {
examples {
features {
feature {
key: "age"
value { float_list: { value: 25 } }
}
feature {
key: "workclass"
value { bytes_list: { value: "Private" } }
}
feature {
key: "education"
value { bytes_list: { value: "11th" } }
}
feature {
key: "education-num"
value { float_list: { value: 7 } }
}
feature {
key: "marital-status"
value { bytes_list: { value: "Never-married" } }
}
feature {
key: "occupation"
value { bytes_list: { value: "Machine-op-inspct" } }
}
feature {
key: "relationship"
value { bytes_list: { value: "Own-child" } }
}
feature {
key: "race"
value { bytes_list: { value: "Black" } }
}
feature {
key: "sex"
value { bytes_list: { value: "Male" } }
}
feature {
key: "capital-gain"
value { float_list: { value: 0 } }
}
feature {
key: "capital-loss"
value { float_list: { value: 0 } }
}
feature {
key: "hours-per-week"
value { float_list: { value: 40 } }
}
feature {
key: "native-country"
value { bytes_list: { value: "United-States" } }
}
}
}
}
}"""
results = local_model_server.make_classification_request(
address, ascii_classification_request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].classes), 2)
self.assertEqual(results[0].classes[0].label, '0')
self.assertLess(results[0].classes[0].score, 0.01)
self.assertEqual(results[0].classes[1].label, '1')
self.assertGreater(results[0].classes[1].score, 0.99)