def main():
training_data = pd.read_csv('../data/20180105_label.csv',
skipinitialspace=True,
engine='python',
dtype=np.float64,
iterator=True,
)
test_data = pd.read_csv('../data/20180107_label.csv',
skipinitialspace=True,
engine='python',
dtype=np.float64,
iterator=True,
)
deep_columns = create_columns(CONTINUOUS_COLUMNS)
model = DNNClassifier(feature_columns=deep_columns,
model_dir='./model',
hidden_units=[10, 10],
n_classes=2,
input_layer_min_slice_size=10000)
tf.logging.set_verbosity(tf.logging.INFO)
training_data_chunk = training_data.get_chunk(1000000000)
model.fit(input_fn=lambda: input_fn(training_data_chunk),
steps=100)
tf.logging.info("end fit model")
test_data_chunk = test_data.get_chunk(10000)
accuracy = model.evaluate(input_fn=lambda: input_fn(test_data_chunk),
steps=100)['accuracy']
print(accuracy * 100)
def main():
# If the training and test sets aren't stored locally, download them.
if not os.path.exists(IRIS_TRAINING):
raw = urlopen(IRIS_TRAINING_URL).read()
with open(IRIS_TRAINING, "wb") as f:
f.write(raw)
if not os.path.exists(IRIS_TEST):
raw = urlopen(IRIS_TEST_URL).read()
with open(IRIS_TEST, "wb") as f:
f.write(raw)
# Load datasets.
training_set = load_csv_with_header(filename=IRIS_TRAINING,
target_dtype=np.int,
features_dtype=np.float32)
test_set = load_csv_with_header(filename=IRIS_TEST,
target_dtype=np.int,
features_dtype=np.float32)
# Specify that all features have real-value data
feature_columns = [real_valued_column("", dimension=4)]
# Build 3 layer DNN with 10, 20, 10 units respectively.
classifier = DNNClassifier(feature_columns=feature_columns,
hidden_units=[10, 20, 10],
n_classes=3,
model_dir="/tmp/iris_model")
# Define the training inputs
def get_train_inputs():
x = tf.constant(training_set.data)
y = tf.constant(training_set.target)
return x, y
# Fit model.
classifier.fit(input_fn=get_train_inputs, steps=2000)
# Define the test inputs
def get_test_inputs():
x = tf.constant(test_set.data)
y = tf.constant(test_set.target)
return x, y
# Evaluate accuracy.
accuracy_score = classifier.evaluate(input_fn=get_test_inputs,
steps=1)["accuracy"]
print("\nTest Accuracy: {0:f}\n".format(accuracy_score))
# Classify two new flower samples.
def new_samples():
return np.array([[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]],
dtype=np.float32)
predictions = list(classifier.predict(input_fn=new_samples))
print("New Samples, Class Predictions: {}\n".format(predictions))
def main(_):
mnist = input_data.read_data_sets("/tmp/data")
X_train = mnist.train.images
X_test = mnist.test.images
Y_train = mnist.train.labels.astype("int")
Y_test = mnist.test.labels.astype("int")
config = RunConfig(tf_random_seed=42, save_checkpoints_secs=10)
feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(
X_train)
validation_monitor = monitors.ValidationMonitor(x=X_test,
y=Y_test,
every_n_steps=100)
dnn_clf = DNNClassifier(
hidden_units=[300, 100],
n_classes=10,
feature_columns=feature_cols,
config=config,
model_dir="/home/mtb/Projects/machine_learning/tensorflow/mnist")
dnn_clf.fit(X_train,
Y_train,
batch_size=50,
steps=4000,
monitors=[validation_monitor])
accuracy_score = dnn_clf.evaluate(x=X_test, y=Y_test)["accuracy"]
print(' accuracy_score: {0} '.format(accuracy_score))
def train_model(item_type):
model_dir = "models/" + item_type.lower().replace(" ", "_")
if os.path.exists(model_dir):
return
print("==> Training model for '%s'" % item_type)
csv_filename = filename = "data/" + item_type.lower().replace(" ", "_") + ".csv"
df_all = pd.read_csv(csv_filename, skipinitialspace=True, encoding='utf-8')
df_all.fillna(0.0, inplace=True)
# Convert the price to a bucket representing a range
df_all['price_chaos'] = (df_all['price_chaos'].apply(util.price_bucket)).astype(int)
# Hash the item type to a number
df_all['itemType'] = (df_all['itemType'].apply(lambda x: util.type_hash[x])).astype(float)
LABEL_COLUMN = util.LABEL_COLUMN
# Split the data 80/20 training/test
percent_test = 20
n = (len(df_all) * percent_test)/100
df_train = df_all.head(len(df_all) - n)
df_test = df_all.tail(n)
train_x = df_train.ix[:, df_train.columns != LABEL_COLUMN].as_matrix().astype(float)
train_y = df_train.as_matrix([LABEL_COLUMN])
test_x = df_test.ix[:, df_test.columns != LABEL_COLUMN].as_matrix().astype(float)
test_y = df_test.as_matrix([LABEL_COLUMN])
deep_columns = tf.contrib.learn.infer_real_valued_columns_from_input(train_x)
hidden_units = util.get_hidden_units(len(df_train.columns)-1)
model = DNNClassifier(model_dir=model_dir, feature_columns=deep_columns, hidden_units=hidden_units,
n_classes=len(util.bins), enable_centered_bias=True)
steps = len(df_train)/75
sessions = (steps/500)+2
for i in range(sessions):
model.fit(train_x, train_y, steps=500, batch_size=5000)
results = model.evaluate(test_x, test_y, steps=1, batch_size=df_test.size)
# Print some predictions from the test data
predictions = df_test.sample(10)
v = model.predict_proba(predictions.ix[:, df_test.columns != LABEL_COLUMN].as_matrix().astype(float), batch_size=10)
price_map = []
for i in v:
# take the top 5 most likely price ranges
top_largest = i.argsort()[-5:][::-1]
prices = {}
for p in top_largest:
prices[util.get_bin_label(p)] = float(round(100*i[p], 1))
price_map.append(prices)
for r in price_map:
print r
def DNNClassifierTrainTask(self, datasource, train_path, test_path, **kwargs):
steps = kwargs.pop("steps", 2000)
if datasource == 'system': # data from system
training_set = load_system_dataset(train_path)
if test_path:
test_set = load_system_dataset(test_path)
feature_columns = [real_valued_column("", dimension=4)]
classifier = DNNClassifier(feature_columns=feature_columns,
**kwargs
# hidden_units=[10, 20, 10],
# n_classes=3
)
if test_path:
classifier.fit(x=training_set.data,
y=training_set.target,
steps=steps)
accuracy_score = classifier.evaluate(x=test_set.data,
y=test_set.target)["accuracy"]
return accuracy_score
def main():
training_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename='./iris_data/iris_training.csv',
target_dtype=np.int,
features_dtype=np.float32)
test_set = tf.contrib.learn.datasets.base.load_csv_with_header(
filename='./iris_data/iris_test.csv',
target_dtype=np.int,
features_dtype=np.float32)
feature_columns = [tf.feature_column.numeric_column("x", shape=[4])]
clf = DNNClassifier(hidden_units=[10, 20, 10],
feature_columns=feature_columns,
model_dir='./iris_model',
n_classes=3)
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": np.array(training_set.data)},
y=np.array(training_set.target),
num_epochs=None,
shuffle=True)
clf.fit(input_fn=train_input_fn, steps=2000)
test_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": np.array(test_set.data)},
y=np.array(test_set.target),
num_epochs=1,
shuffle=False)
accuracy_score = clf.evaluate(input_fn=test_input_fn)["accuracy"]
print("\nTest Accuracy: {0:f}\n".format(accuracy_score))
new_samples = np.array([[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]],
dtype=np.float32)
predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": new_samples},
num_epochs=1,
shuffle=False)
predictions = list(clf.predict(input_fn=predict_input_fn))
print predictions
print("New Samples, Class Predictions: {}\n".format(predictions))
class DNN(BaseEstimator, ClassifierMixin):
def __init__(self,
n_classes,
type="w2v",
hidden_units=[10, 20, 10],
num_features=100,
context=10,
method=1):
#if type=="w2v":
#self.model = w2v_helpers.get_word2vec(num_features, context, method)
self.type = type
self.classifier = DNNClassifier(hidden_units=hidden_units,
n_classes=n_classes)
def pre_transformX(self, df, colnames, df_test=None, n_gram=None):
data = None
if self.type == "w2v":
data = features_helpers.create_sentences(df, colnames)
data = features_helpers.transform_to_w2v_sentences(
data, self.model)
return data.as_matrix()
else:
x_train, x_test = features_helpers.transform_to_bow(
df, df_test, colnames, n_gram)
return x_train, x_test
def pre_transformY(self, df, list_dict):
y = map(lambda w: list_dict.index(w), list(df))
return np.array(y)
def fit(self, X, y=None):
self.classifier.fit(x=X, y=y, steps=200)
def predict(self, X, y=None):
return self.classifier.predict(X)
def evaluate(self, X, Y):
return self.classifier.evaluate(x=X, y=Y)["accuracy"]
def score(self, X, y, sample_weight=None):
return super(DNN, self).score(X, y, sample_weight)