def input_fn():
features = layers.create_feature_spec_for_parsing(
get_feature_column(mode))
feature_map = learn.read_batch_features(
file_pattern = input_files,
batch_size = batch_size,
features = features,
reader = tf.TFRecordReader,
randomize_input = True,
num_epochs = num_epochs,
queue_capacity = 200000 + batch_size * 5,
name = 'read_batch_features_{}'.format(mode))
if mode == learn.ModeKeys.TRAIN:
target = feature_map.pop('label')
labels = {'labels':target}
else:
# NOTE: sample count of the last batch maybe less than batch_size
# thus batch_size should be adjusted to the exact value here
exact_batch_shape = feature_map['res_len']
# construct labels for recall/precision metrics
recall_target = tf.zeros_like(exact_batch_shape, dtype = tf.int64)
# construct labels for accuracy metrics
accuracy_target = [tf.ones_like(exact_batch_shape, dtype = tf.int64)]
for i in xrange(DISTRACTOR_COUNT):
accuracy_target.append(
tf.zeros_like(exact_batch_shape, dtype = tf.int64))
accuracy_target = tf.concat(values = accuracy_target, axis = 0)
labels = {'labels': accuracy_target, 'recall_labels': recall_target}
#print('feature_map=%s' % (feature_map))
#print('labels=%s' % (labels))
return feature_map, labels
def main():
if not os.path.exists(TRAINING_DATA):
print("ERR: training data file does not exist.")
sys.exit(1)
if not os.path.exists(TEST_DATA):
print("ERR: test data file does not exist.")
sys.exit(1)
training_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename=TRAINING_DATA, target_dtype=np.int, features_dtype=np.float32)
test_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename=TEST_DATA, target_dtype=np.int, features_dtype=np.float32)
feature_columns = [
tf.contrib.layers.real_valued_column("", dimension=FEATURES_NO)
]
classifier = tf.contrib.learn.DNNClassifier(
feature_columns=feature_columns,
hidden_units=[10, 20, 10],
n_classes=CLASSES_NO,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1))
def training_get_inputs():
x = tf.constant(training_set.data)
y = tf.constant(training_set.target)
return x, y
classifier.fit(input_fn=training_get_inputs, steps=100000)
def test_get_inputs():
x = tf.constant(test_set.data)
y = tf.constant(test_set.target)
return x, y
accuracy_score = classifier.evaluate(input_fn=test_get_inputs,
steps=1)["accuracy"]
print(("\nAccuracy: {0:f}\n".format(accuracy_score)))
def classify_sample():
return np.array([[1077, 0.49, 10.33, 10.52], [346, 0.95, 19.37, 9.89]],
dtype=np.float32)
predictions = list(classifier.predict_classes(input_fn=classify_sample))
predictions = [label_classify(i) for i in predictions]
print(" ************* Examples of classification *************")
print("Prediction: \t{}".format(predictions))
print("Output: \t{}\n".format([label_classify(2), label_classify(0)]))
tfrecord_serving_input_fn = build_parsing_serving_input_fn(
create_feature_spec_for_parsing(feature_columns))
classifier.export_savedmodel(export_dir_base=MODEL_DIR_TRAINED,
serving_input_fn=tfrecord_serving_input_fn,
as_text=False)
def _maybe_export(self):
"""执行模型导出"""
if self._model_export_spec is None:
return
feature_spec = create_feature_spec_for_parsing(self._model_export_spec.features)
serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec)
tmp_export_model_dir = tempfile.mkdtemp()
tmp_export_model_path = self._estimator.export_savedmodel(
tmp_export_model_dir,
serving_input_fn)
print("temp export model path: {}".format(tmp_export_model_path))
os.rename(tmp_export_model_path, self._model_export_spec.export_dir)
print('Succeed to rename "{0}" to "{1}"'.format(tmp_export_model_path, self._model_export_spec.export_dir))
def _get_feature_ops_from_example(self, examples_batch):
column_types = layers.create_feature_spec_for_parsing(
(self._get_linear_feature_columns() or []) +
(self._get_dnn_feature_columns() or []))
features = parsing_ops.parse_example(examples_batch, column_types)
return features
def _get_feature_ops_from_example(self, examples_batch):
column_types = layers.create_feature_spec_for_parsing(
(self._get_linear_feature_columns() or []) + (self._get_dnn_feature_columns() or [])
)
features = parsing_ops.parse_example(examples_batch, column_types)
return features
def save_tf_learn_model(estimator, model_name, export_dir, feature_columns, ):
feature_spec = create_feature_spec_for_parsing(feature_columns)
serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec)
export_dir = os.path.join(export_dir, model_name)
estimator.export_savedmodel(export_dir, serving_input_fn)
print("Done exporting tf.learn model to " + export_dir + "!")
def main():
# If the training and test sets aren't stored locally, abort mission
if not os.path.exists(MEDICAID_TRAINING):
print("error: no training data")
sys.exit(1)
if not os.path.exists(MEDICAID_TEST):
print("error: no test data")
sys.exit(1)
# Load datasets.
training_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename=MEDICAID_TRAINING,
target_dtype=np.int,
features_dtype=np.float32)
test_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename=MEDICAID_TEST, target_dtype=np.int, features_dtype=np.float32)
# Specify that all features have real-value data
feature_columns = [
tf.contrib.layers.real_valued_column("", dimension=NUMBER_FEATURES)
]
# Build 3 layer DNN (classifier) with 10, 20, 10 hidden units respectively.
classifier = tf.contrib.learn.DNNClassifier(
feature_columns=feature_columns,
hidden_units=[10, 20, 10],
n_classes=NUMBER_CLASSES,
model_dir=MEDICAID_MODEL_DIR,
config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1))
# Define the training inputs
def get_train_inputs():
x = tf.constant(training_set.data)
y = tf.constant(training_set.target)
return x, y
# Fit model.
classifier.fit(input_fn=get_train_inputs, steps=100000)
# Define the test inputs
def get_test_inputs():
x = tf.constant(test_set.data)
y = tf.constant(test_set.target)
return x, y
# Evaluate the accuracy of the model after training accuracy.
accuracy_score = classifier.evaluate(input_fn=get_test_inputs,
steps=1)["accuracy"]
print(("\nAccuracy: {0:f}\n".format(accuracy_score)))
# Classify new samples, one of each event type, a walk and a fall
# TODO: Blake - these samples are just rounded values of some random rows
# in the test set.
def new_samples():
return np.array([[1077, 0.49, 10.33, 10.52], [346, 0.95, 19.37, 9.89]],
dtype=np.float32)
predictions = list(classifier.predict_classes(input_fn=new_samples))
predictions = [class_to_label(i) for i in predictions]
# Classify some samples, just to get a feel for the model success.
print("==== Sample Classifications ====")
print("Prediction: \t{}".format(predictions))
print("Answer: \t{}\n".format([class_to_label(2), class_to_label(0)]))
# Save the model to a binary .pb file for loading into Java (on Android)
# See: https://github.com/tensorflow/tensorflow/issues/3340
# TensorFlow is REALLY poorly documented. This seems to work. I don't
# think anyone actually knows why.
# Actually this dude got somewhere:
# https://github.com/MtDersvan/tf_playground/blob/master/wide_and_deep_tutorial/wide_and_deep_basic_serving.md
tfrecord_serving_input_fn = build_parsing_serving_input_fn(
create_feature_spec_for_parsing(feature_columns))
classifier.export_savedmodel(export_dir_base=MEDICAID_MODEL_DIR_TRAINED,
serving_input_fn=tfrecord_serving_input_fn,
as_text=False)
# train_inputs, test_inputs, train_output, test_output = train_test_split(total_inputs, total_output, test_size=0.2, random_state=42)
# feature_columns = [tf.contrib.layers.real_valued_column("", dimension=train_inputs.shape[1],dtype=int)]
feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)]
#target_column = [tf.contrib.layers.real_valued_column("output", dimension=train_output.shape[1])]
classifier = learn.DNNClassifier(hidden_units=[10, 20, 5],
n_classes=5,
feature_columns=feature_columns)
classifier.fit(x=100, y=50, steps=100)
#Save Model into saved_model.pbtxt file (possible to Load in Java)
tfrecord_serving_input_fn = tf.contrib.learn.build_parsing_serving_input_fn(
layers.create_feature_spec_for_parsing(feature_columns))
classifier.export_savedmodel(export_dir_base="test",
serving_input_fn=tfrecord_serving_input_fn,
as_text=True)
# Measure accuracy
pred = list(classifier.predict(test_inputs, as_iterable=True))
score = metrics.accuracy_score(test_output, pred)
print("Final score: {}".format(score))
# test individual samples
sample_1 = np.array([[2500, 200, 3, 50], [900, 200, 4, 60]], dtype=int)
# sample_2 = np.array( [[1.0,0.7982741870963959,1.0,-0.46270838239235024,0.040320274521029376,0.443451913224413,-1.0,1.0,1.0,-1.0,0.36689718911339564,-0.13577379160035796,-0.5162916256414466,-0.03373651520104648,1.0,1.0,1.0,1.0,0.786999801054777,-0.43856035121103853,-0.8199093927945158,1.0,-1.0,-1.0,-0.1134921695894473,-1.0,0.6420892436196663,0.7871737734493178,1.0,0.6501788845358409,1.0,1.0,1.0,-0.17586627413625022,0.8817194210401085]], dtype=float)
pred = list(classifier.predict(sample_2, as_iterable=True))
print("Prediction for sample_1 is:{} ".format(pred))
#every_n_steps=200, # when to run the monitor - not working - forcing with save_checkpoints_steps
early_stopping_metric="accuracy", #"accuracy" or "loss"
early_stopping_metric_minimize=False, #False Maximize accuracy (True is minimize applied to loss)
early_stopping_rounds=early_stopping_rounds)
print("Fit will start...")
#classifier = learn.SKCompat(classifier) # For Sklearn compatibility
classifier.fit(train_inputs, train_output, steps=training_steps ,
batch_size=batch_size,
monitors=[validation_monitor])
print("Fit is finish...")
#print (classifier.get_variable_names())
#Save Model into saved_model.pbtxt file (possible to Load in Java)
tfrecord_serving_input_fn = tf.contrib.learn.build_parsing_serving_input_fn(layers.create_feature_spec_for_parsing(feature_columns))
classifier.export_savedmodel(export_dir_base="test", serving_input_fn = tfrecord_serving_input_fn,as_text=True)
# Measure accuracy
pred = list(classifier.predict(test_inputs, as_iterable=True))
#pred = list(classifier.predict(test_inputs))
score = metrics.accuracy_score(test_output, pred)
print("Final score: {}".format(score))
accuracy_score = classifier.evaluate(x=test_inputs,
y=test_output)["accuracy"]
print('Accuracy: {0:f}'.format(accuracy_score))
# test individual samples
# TensorFlow provides many helper methods to read and set up data sets.
# Here we use `numpy_input_fn`. We have to tell the function how many batches
# of data (num_epochs) we want and how big each batch should be.
x = np.array([1., 2., 3., 4.])
y = np.array([0., -1., -2., -3.])
input_fn = tf.contrib.learn.io.numpy_input_fn({"x": x},
y,
batch_size=4,
num_epochs=1000)
# We can invoke 1000 training steps by invoking the `fit` method and passing the
# training data set.
estimator.fit(input_fn=input_fn, steps=1000)
# Here we evaluate how well our model did. In a real example, we would want
# to use a separate validation and testing data set to avoid overfitting.
print(estimator.evaluate(input_fn=input_fn))
# export Estimator
feature_spec = create_feature_spec_for_parsing(features)
serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec)
export_model_dir = tempfile.mkdtemp()
export_model_path = estimator.export_savedmodel(export_model_dir,
serving_input_fn)
print("exported model path: {}".format(export_model_path))
# def predict_input_fn():
# return {'x': tf.constant([1.0, 2.0])}
# print(list(estimator.predict(input_fn=predict_input_fn)))
