def input_fn(data_dir, params):
dataset = dg.data(
"udacity-selfdriving-simulator",
path=data_dir,
)
dataset = dataset.get()
df = dataset.df
df = process_dataframe(df, params)
if params.only_center_camera:
df = df[df.camera == 1]
df = dg.shuffle(df)
tensors = dict(
filepath=df.filepath.as_matrix(),
steering=df.steering.as_matrix(),
camera=df.camera.as_matrix(),
original_steering=df.original_steering.as_matrix(),
)
if "flipped" in df:
tensors["flipped"] = df.flipped.as_matrix().astype(np.int32)
ds = tf.data.Dataset.from_tensor_slices(tensors)
ds = ds.apply(
tf.contrib.data.shuffle_and_repeat(buffer_size=params.buffer_size, ))
ds = ds.apply(
tf.contrib.data.map_and_batch(
lambda row: process_data(row, params),
batch_size=params.batch_size,
num_parallel_calls=params.n_threads,
drop_remainder=True,
))
ds = ds.prefetch(tf.contrib.data.AUTOTUNE)
return ds
def main(device):
n_classes = data("visual-words").n_classes
graph = tf.Graph()
sess = tf.Session(graph=graph)
# inputs
inputs_t, model_t = get_templates(n_classes)
# model
with tf.device(device):
inputs = inputs_t()
model = model_t(inputs)
with graph.as_default():
print("")
print("##########################################################")
print("Number of Weights = {:,}".format(model.count_weights()))
print("##########################################################")
def main(device):
print("DEVICE:", device)
# seed: resultados repetibles
seed = 32
np.random.seed(seed=seed)
random.seed(seed)
# dataget
dataset = data("german-traffic-signs").get()
# obtener imagenes
print("loading data")
features_test, labels_test = dataset.test_set.arrays()
# features_test, labels_test = next(dataset.test_set.random_batch_arrays_generator(500))
#model
with tf.device(device):
graph = tf.Graph()
sess = tf.Session(graph=graph)
# inputs
inputs = SupervisedInputs(
name=network_name + "_inputs",
graph=graph,
sess=sess,
# tensors
features=dict(shape=(None, 32, 32, 3)),
labels=dict(shape=(None, ), dtype=tf.uint8))
# create model template
template = Model(
n_classes=43,
name=network_name,
model_path=model_path,
graph=graph,
sess=sess,
seed=seed,
)
inputs = inputs()
model = template(inputs)
# restore
print("restoring model")
model.initialize(restore=True)
# test
print("testing")
generator = batch_generator(len(features_test), 100)
generator = map(
lambda batch: dict(features=features_test[batch],
labels=labels_test[batch]), generator)
predictions = model.batch_predict(
generator,
print_fn=lambda batch: print(
accuracy_score(np.argmax(model.predict(**batch), axis=1),
batch["labels"]),
np.mean(
np.argmax(model.predict(**batch), axis=1) == batch[
"labels"]), model.score(**batch)))
predictions = np.argmax(predictions, axis=1)
test_score = accuracy_score(predictions, labels_test)
print("test score: {}".format(test_score))
def main(device, epochs, batch_size, loss, restore):
# seed: resultados repetibles
seed = 32
np.random.seed(seed=seed)
random.seed(seed)
# dataget
dataset = data("visual-words").get(process=False)
df = dataset.training_set.dataframe()
dataset.training_set._dataframe = df[df["image"].apply(lambda a: a.shape)
== (299, 299, 3)]
# utils.process_steering(dataset.training_set, params.alpha, params.straight_tol, params.steering_filter)
# print(dataset.training_set.pure_dataframe().columns)
# obtener todas las imagenes (lento)
def data_generator_fn():
data_generator = dataset.training_set.random_batch_arrays_generator(
batch_size)
data_generator = cz.map(
Dict(image=P[0], labels=P[1], keras_training=False),
data_generator)
# data_generator = cz.map(utils.get_processed_image, data_generator)
return data_generator
graph = tf.Graph()
sess = tf.Session(graph=graph)
# inputs
input_t, model_t = get_templates(dataset.n_classes, seed=seed)
# model
with tf.device(device):
inputs = input_t()
model = model_t(inputs)
# initialize variables
print("Initializing Model: restore = {}".format(restore))
model.initialize(restore=restore)
# start queue
# inputs.start_queue(data_generator_fn)
# fit
print("training")
model.fit(
data_generator=data_generator_fn(),
epochs=epochs,
# log_summaries = True,
log_interval=10,
print_test_info=True,
on_train=[
dict(when=lambda step, **kwargs: step % 1000 == 0,
do=lambda **kwargs: model.save())
])
# save
print("saving model")
model.save()
def main(device, epochs, batch_size):
# seed: resultados repetibles
seed = 32
np.random.seed(seed=seed)
random.seed(seed)
# dataget
dataset = data("german-traffic-signs").get()
# data_generator
data_generator = dataset.training_set.random_batch_arrays_generator(
batch_size)
data_generator = utils.batch_random_image_rotation(data_generator, 15.0)
data_generator = cz.map(Dict(features=P[0], labels=P[1]), data_generator)
graph = tf.Graph()
sess = tf.Session(graph=graph)
# inputs
inputs = SupervisedInputs(
name=network_name + "_inputs",
graph=graph,
sess=sess,
# tensors
features=dict(shape=(None, 32, 32, 3)),
labels=dict(shape=(None, ), dtype=tf.uint8))
# create model template
template = Model(
n_classes=43,
name=network_name,
model_path=model_path,
graph=graph,
sess=sess,
seed=seed,
optimizer=tf.train.AdamOptimizer,
)
# model
with tf.device(device):
inputs = inputs()
model = template(inputs)
# initialize variables
model.initialize()
# fit
print("training")
model.fit(
data_generator=data_generator,
epochs=epochs,
log_summaries=True,
log_interval=10,
print_test_info=True,
)
# save
print("saving model")
model.save()
#MNIST
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from time import time
from sklearn import datasets, svm, metrics
from sklearn.datasets import fetch_mldata
from numpy import arange
from sklearn.model_selection import train_test_split
from dataget import data
#Read image input as vector or matrix
#Train algorithm using data from data/training-set
#Get data
dataset = data("mnist").get()
#Format training_set
features_train, labels_train = dataset.training_set.arrays()
features_train = features_train.reshape((60000, 28 * 28))
#print("Features shape: {} \nLabels shape: {}".format(features_train.shape, labels_train.shape))
#Format test_set
features_test, labels_test = dataset.test_set.arrays()
features_test = features_test.reshape((10000, 28 * 28))
#print("Features shape: {} \nLabels shape: {}".format(features_test.shape, labels_test.shape))
def kClassify(features_train, labels_train):
clf = KNeighborsClassifier(n_neighbors=2, algorithm='brute', leaf_size=7)
t0 = time()
clf.fit(features_train, labels_train)
print "training time:", round(time() - t0, 3), "s"
import dataget as dg
ds = dg.data(
"udacity-selfdriving-simulator",
path="data/raw",
)
ds = ds.get(download=False)
df = ds.df
print(df.head())
print(df.filepath.iloc[0])