def input_fn(): features = layers.create_feature_spec_for_parsing( get_feature_column(mode)) feature_map = learn.read_batch_features( file_pattern = input_files, batch_size = batch_size, features = features, reader = tf.TFRecordReader, randomize_input = True, num_epochs = num_epochs, queue_capacity = 200000 + batch_size * 5, name = 'read_batch_features_{}'.format(mode)) if mode == learn.ModeKeys.TRAIN: target = feature_map.pop('label') labels = {'labels':target} else: # NOTE: sample count of the last batch maybe less than batch_size # thus batch_size should be adjusted to the exact value here exact_batch_shape = feature_map['res_len'] # construct labels for recall/precision metrics recall_target = tf.zeros_like(exact_batch_shape, dtype = tf.int64) # construct labels for accuracy metrics accuracy_target = [tf.ones_like(exact_batch_shape, dtype = tf.int64)] for i in xrange(DISTRACTOR_COUNT): accuracy_target.append( tf.zeros_like(exact_batch_shape, dtype = tf.int64)) accuracy_target = tf.concat(values = accuracy_target, axis = 0) labels = {'labels': accuracy_target, 'recall_labels': recall_target} #print('feature_map=%s' % (feature_map)) #print('labels=%s' % (labels)) return feature_map, labels
def main(): if not os.path.exists(TRAINING_DATA): print("ERR: training data file does not exist.") sys.exit(1) if not os.path.exists(TEST_DATA): print("ERR: test data file does not exist.") sys.exit(1) training_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=TRAINING_DATA, target_dtype=np.int, features_dtype=np.float32) test_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=TEST_DATA, target_dtype=np.int, features_dtype=np.float32) feature_columns = [ tf.contrib.layers.real_valued_column("", dimension=FEATURES_NO) ] classifier = tf.contrib.learn.DNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=CLASSES_NO, model_dir=MODEL_DIR, config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) def training_get_inputs(): x = tf.constant(training_set.data) y = tf.constant(training_set.target) return x, y classifier.fit(input_fn=training_get_inputs, steps=100000) def test_get_inputs(): x = tf.constant(test_set.data) y = tf.constant(test_set.target) return x, y accuracy_score = classifier.evaluate(input_fn=test_get_inputs, steps=1)["accuracy"] print(("\nAccuracy: {0:f}\n".format(accuracy_score))) def classify_sample(): return np.array([[1077, 0.49, 10.33, 10.52], [346, 0.95, 19.37, 9.89]], dtype=np.float32) predictions = list(classifier.predict_classes(input_fn=classify_sample)) predictions = [label_classify(i) for i in predictions] print(" ************* Examples of classification *************") print("Prediction: \t{}".format(predictions)) print("Output: \t{}\n".format([label_classify(2), label_classify(0)])) tfrecord_serving_input_fn = build_parsing_serving_input_fn( create_feature_spec_for_parsing(feature_columns)) classifier.export_savedmodel(export_dir_base=MODEL_DIR_TRAINED, serving_input_fn=tfrecord_serving_input_fn, as_text=False)
def _maybe_export(self): """执行模型导出""" if self._model_export_spec is None: return feature_spec = create_feature_spec_for_parsing(self._model_export_spec.features) serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec) tmp_export_model_dir = tempfile.mkdtemp() tmp_export_model_path = self._estimator.export_savedmodel( tmp_export_model_dir, serving_input_fn) print("temp export model path: {}".format(tmp_export_model_path)) os.rename(tmp_export_model_path, self._model_export_spec.export_dir) print('Succeed to rename "{0}" to "{1}"'.format(tmp_export_model_path, self._model_export_spec.export_dir))
def _get_feature_ops_from_example(self, examples_batch): column_types = layers.create_feature_spec_for_parsing( (self._get_linear_feature_columns() or []) + (self._get_dnn_feature_columns() or [])) features = parsing_ops.parse_example(examples_batch, column_types) return features
def _get_feature_ops_from_example(self, examples_batch): column_types = layers.create_feature_spec_for_parsing( (self._get_linear_feature_columns() or []) + (self._get_dnn_feature_columns() or []) ) features = parsing_ops.parse_example(examples_batch, column_types) return features
def save_tf_learn_model(estimator, model_name, export_dir, feature_columns, ): feature_spec = create_feature_spec_for_parsing(feature_columns) serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec) export_dir = os.path.join(export_dir, model_name) estimator.export_savedmodel(export_dir, serving_input_fn) print("Done exporting tf.learn model to " + export_dir + "!")
def main(): # If the training and test sets aren't stored locally, abort mission if not os.path.exists(MEDICAID_TRAINING): print("error: no training data") sys.exit(1) if not os.path.exists(MEDICAID_TEST): print("error: no test data") sys.exit(1) # Load datasets. training_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=MEDICAID_TRAINING, target_dtype=np.int, features_dtype=np.float32) test_set = tf.contrib.learn.datasets.base.load_csv_with_header( filename=MEDICAID_TEST, target_dtype=np.int, features_dtype=np.float32) # Specify that all features have real-value data feature_columns = [ tf.contrib.layers.real_valued_column("", dimension=NUMBER_FEATURES) ] # Build 3 layer DNN (classifier) with 10, 20, 10 hidden units respectively. classifier = tf.contrib.learn.DNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=NUMBER_CLASSES, model_dir=MEDICAID_MODEL_DIR, config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1)) # Define the training inputs def get_train_inputs(): x = tf.constant(training_set.data) y = tf.constant(training_set.target) return x, y # Fit model. classifier.fit(input_fn=get_train_inputs, steps=100000) # Define the test inputs def get_test_inputs(): x = tf.constant(test_set.data) y = tf.constant(test_set.target) return x, y # Evaluate the accuracy of the model after training accuracy. accuracy_score = classifier.evaluate(input_fn=get_test_inputs, steps=1)["accuracy"] print(("\nAccuracy: {0:f}\n".format(accuracy_score))) # Classify new samples, one of each event type, a walk and a fall # TODO: Blake - these samples are just rounded values of some random rows # in the test set. def new_samples(): return np.array([[1077, 0.49, 10.33, 10.52], [346, 0.95, 19.37, 9.89]], dtype=np.float32) predictions = list(classifier.predict_classes(input_fn=new_samples)) predictions = [class_to_label(i) for i in predictions] # Classify some samples, just to get a feel for the model success. print("==== Sample Classifications ====") print("Prediction: \t{}".format(predictions)) print("Answer: \t{}\n".format([class_to_label(2), class_to_label(0)])) # Save the model to a binary .pb file for loading into Java (on Android) # See: https://github.com/tensorflow/tensorflow/issues/3340 # TensorFlow is REALLY poorly documented. This seems to work. I don't # think anyone actually knows why. # Actually this dude got somewhere: # https://github.com/MtDersvan/tf_playground/blob/master/wide_and_deep_tutorial/wide_and_deep_basic_serving.md tfrecord_serving_input_fn = build_parsing_serving_input_fn( create_feature_spec_for_parsing(feature_columns)) classifier.export_savedmodel(export_dir_base=MEDICAID_MODEL_DIR_TRAINED, serving_input_fn=tfrecord_serving_input_fn, as_text=False)
# train_inputs, test_inputs, train_output, test_output = train_test_split(total_inputs, total_output, test_size=0.2, random_state=42) # feature_columns = [tf.contrib.layers.real_valued_column("", dimension=train_inputs.shape[1],dtype=int)] feature_columns = [tf.contrib.layers.real_valued_column("", dimension=4)] #target_column = [tf.contrib.layers.real_valued_column("output", dimension=train_output.shape[1])] classifier = learn.DNNClassifier(hidden_units=[10, 20, 5], n_classes=5, feature_columns=feature_columns) classifier.fit(x=100, y=50, steps=100) #Save Model into saved_model.pbtxt file (possible to Load in Java) tfrecord_serving_input_fn = tf.contrib.learn.build_parsing_serving_input_fn( layers.create_feature_spec_for_parsing(feature_columns)) classifier.export_savedmodel(export_dir_base="test", serving_input_fn=tfrecord_serving_input_fn, as_text=True) # Measure accuracy pred = list(classifier.predict(test_inputs, as_iterable=True)) score = metrics.accuracy_score(test_output, pred) print("Final score: {}".format(score)) # test individual samples sample_1 = np.array([[2500, 200, 3, 50], [900, 200, 4, 60]], dtype=int) # sample_2 = np.array( [[1.0,0.7982741870963959,1.0,-0.46270838239235024,0.040320274521029376,0.443451913224413,-1.0,1.0,1.0,-1.0,0.36689718911339564,-0.13577379160035796,-0.5162916256414466,-0.03373651520104648,1.0,1.0,1.0,1.0,0.786999801054777,-0.43856035121103853,-0.8199093927945158,1.0,-1.0,-1.0,-0.1134921695894473,-1.0,0.6420892436196663,0.7871737734493178,1.0,0.6501788845358409,1.0,1.0,1.0,-0.17586627413625022,0.8817194210401085]], dtype=float) pred = list(classifier.predict(sample_2, as_iterable=True)) print("Prediction for sample_1 is:{} ".format(pred))
#every_n_steps=200, # when to run the monitor - not working - forcing with save_checkpoints_steps early_stopping_metric="accuracy", #"accuracy" or "loss" early_stopping_metric_minimize=False, #False Maximize accuracy (True is minimize applied to loss) early_stopping_rounds=early_stopping_rounds) print("Fit will start...") #classifier = learn.SKCompat(classifier) # For Sklearn compatibility classifier.fit(train_inputs, train_output, steps=training_steps , batch_size=batch_size, monitors=[validation_monitor]) print("Fit is finish...") #print (classifier.get_variable_names()) #Save Model into saved_model.pbtxt file (possible to Load in Java) tfrecord_serving_input_fn = tf.contrib.learn.build_parsing_serving_input_fn(layers.create_feature_spec_for_parsing(feature_columns)) classifier.export_savedmodel(export_dir_base="test", serving_input_fn = tfrecord_serving_input_fn,as_text=True) # Measure accuracy pred = list(classifier.predict(test_inputs, as_iterable=True)) #pred = list(classifier.predict(test_inputs)) score = metrics.accuracy_score(test_output, pred) print("Final score: {}".format(score)) accuracy_score = classifier.evaluate(x=test_inputs, y=test_output)["accuracy"] print('Accuracy: {0:f}'.format(accuracy_score)) # test individual samples
# TensorFlow provides many helper methods to read and set up data sets. # Here we use `numpy_input_fn`. We have to tell the function how many batches # of data (num_epochs) we want and how big each batch should be. x = np.array([1., 2., 3., 4.]) y = np.array([0., -1., -2., -3.]) input_fn = tf.contrib.learn.io.numpy_input_fn({"x": x}, y, batch_size=4, num_epochs=1000) # We can invoke 1000 training steps by invoking the `fit` method and passing the # training data set. estimator.fit(input_fn=input_fn, steps=1000) # Here we evaluate how well our model did. In a real example, we would want # to use a separate validation and testing data set to avoid overfitting. print(estimator.evaluate(input_fn=input_fn)) # export Estimator feature_spec = create_feature_spec_for_parsing(features) serving_input_fn = input_fn_utils.build_parsing_serving_input_fn(feature_spec) export_model_dir = tempfile.mkdtemp() export_model_path = estimator.export_savedmodel(export_model_dir, serving_input_fn) print("exported model path: {}".format(export_model_path)) # def predict_input_fn(): # return {'x': tf.constant([1.0, 2.0])} # print(list(estimator.predict(input_fn=predict_input_fn)))