def evaluation():
cat_dir = 'D:\\machine_learning_python\\tensorflow_classification\\PetImages\\Cat\\'
dog_dir = 'D:\\machine_learning_python\\tensorflow_classification\\PetImages\\Dog\\'
logs_train_dir = 'D:\\machine_learning_python\\tensorflow_classification\\records\\'
train_images, train_labels, eval_images, eval_labels = data_preparation.get_data(
cat_dir, dog_dir)
iter = int(len(eval_images) / BATCH_SIZE)
eval_batch, eval_label_batch = data_preparation.create_batch(
eval_images, eval_labels, IM_W, IM_H, BATCH_SIZE, CAPACITY)
logits = model_generation.inference(eval_batch, BATCH_SIZE, N_CLASSES)
accu_batch = model_generation.evaluation_batch(logits, eval_label_batch)
saver = tf.strain.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(logs_train_dir)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is $s' % global_step)
else:
print('No checkpoint file fouund')
accuracy = 0.0
for i in range(iter):
accuracy += sess.run(accu_batch)
accuracy = accuracy / iter
print('Accuracy is about %.3f' % (accuracy))
def run(cfg: DictConfig) -> None:
# Getting data
data = get_data(dataname=cfg.stock.name, period=cfg.grain.period)
# Run optimization
dct = bayesian_optimization(data, cfg)
# Generate report
generate_report(dct, cfg)
def run_training():
cat_dir = 'D:\\machine_learning_python\\tensorflow_classification\\PetImages\\Cat\\'
dog_dir = 'D:\\machine_learning_python\\tensorflow_classification\\PetImages\\Dog\\'
logs_train_dir = 'D:\\machine_learning_python\\tensorflow_classification\\records\\'
train_images, train_labels, eval_images, eval_labels = data_preparation.get_data(
cat_dir, dog_dir)
train_batch, train_label_batch = data_preparation.create_batch(
train_images, train_labels, IM_W, IM_H, BATCH_SIZE, CAPACITY)
train_logits = model_generation.inference(train_batch, BATCH_SIZE,
N_CLASSES)
train_loss = model_generation.loss(train_logits, train_label_batch)
train_op = model_generation.training(train_loss, LEARNING_RATE)
train_acc = model_generation.evaluation_batch(train_logits,
train_label_batch)
summary_op = tf.summary.merge_all()
sess = tf.Session()
train_writer = tf.summary.FileWriter(logs_train_dir, sess.graph)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
print('number of threads is %d' % (len(threads)))
try:
for step in np.arange(MAX_STEP):
if coord.should_stop():
break
_, tra_loss, tra_acc = sess.run([train_op, train_loss, train_acc])
if step % 10 == 0:
print('Step %d, train loss = %.2f, train accuracy = %.2f%%' %
(step, tra_loss, tra_acc))
summary_str = sess.run(summary_op)
train_writer.add_summary(summary_str, step)
if step % 2000 == 0 or (step + 1) == MAX_STEP:
checkpoint_path = os.path.join(logs_train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
except tf.errors.OutOfRangeError:
print('Training is done!')
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def experiment(cfg: DictConfig) -> None:
# Reading dataset
X, y = get_data()
# Splitting data
X_train, X_test, y_train, y_test = split(X, y, 0.2)
#Building pipeline
pipe = build_pipeline(cfg)
#Hyperparameter optimization & Dump best estimator
model = hpo(pipe, X_train, y_train, cfg)
#Evaluation report
generate_report(y_train, model.predict(X_train), y_test,
model.predict(X_test))
def contrastive_loss(y_true, y_pred):
margin = 1
return K.mean(y_true * K.square(y_pred) +
(1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def compute_accuracy(predictions, labels):
return labels[predictions.ravel() < 0.5].mean()
size = 2
total_sample_size = 10000
X, Y = get_data(size, total_sample_size)
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.25)
input_dim = x_train.shape[2:]
img_a = Input(shape=input_dim)
img_b = Input(shape=input_dim)
base_network = build_base_network(input_dim)
feat_vect_a = base_network(img_a)
feat_vect_b = base_network(img_b)
distance = Lambda(euclidean_distance, output_shape=eucl_dist_out_shapes)(
[feat_vect_a, feat_vect_b])
epochs = 15
rms = RMSprop()
def __init__(
self,
data_path,
path_to_labels_file,
data_shapes,
batch_size,
max_size="",
is_prepared=False,
tf_record_file_path="",
augment_data=False,
validation_split=0.0,
):
"""
:param data_path: Path to image data
:param path_to_labels_file: Path to csv file with image labels
:param batch_size: Batch size
:param max_size: The maximum value of images used in the generator. Optional parameter.
:param is_prepared: If value is False, generator creates one file containing all images as tf_record file.
:param tf_record_file_path: Path to tf_record file.
:param augment_data: True if you want to perform augmentation on images.
"""
if tf_record_file_path:
self.tf_record_file_name = tf_record_file_path
else:
self.tf_record_file_name = data_path + "tf_record_dataset.tfrecord"
self.image_shape = data_shapes["input"]
self.label_shape = data_shapes["output"]
self.augment_data = augment_data
self.validation_split = validation_split
# TODO change that
self.images = data_preparation.get_data(
data_path,
data_preparation.folders) # + data_preparation.get_validation()
self.labels = data_preparation.labels_generation(
data_path, path_to_labels_file)
# self.labels = self._load_image_labels(path_to_labels_file)
self.batch_size = batch_size
self.dataset_size = (min(len(self.images), max_size)
if max_size else len(self.images))
if not is_prepared:
self.prepare_dataset()
dbg(f"Loaded number of entries: {self.dataset_size}")
if self.augment_data:
dbg(
"Data augmentation won't work\n"
"We need to change the imgaug to another library which is based on"
"TF operation graph or invoke tf.session here which will cause lot"
"of problems",
mode="crit",
)
self.image_preprocessor = ImagePreprocessor()
# create dataset object
tfdataset = tf.data.TFRecordDataset(
filenames=[self.tf_record_file_name])
tfdataset = tfdataset.shuffle(
256) # this might be a performace problem
# if we want validation data, take 'validation_split' size out of dataset
if self.validation_split:
sizee = math.ceil(self.validation_split * self.dataset_size)
self.train_data = self.images[:sizee]
self.validation_data = self.images[sizee:]
self.tfdataset_validation = tfdataset.take(
math.ceil(self.validation_split * self.dataset_size))
self.tfdataset_validation = self.tfdataset_validation.map(
self._parse_function)
self.tfdataset_validation = self.tfdataset_validation.batch(
self.batch_size)
self.tfdataset_validation = self.tfdataset_validation.prefetch(
tf.contrib.data.AUTOTUNE)
self.tfdataset_validation = self.tfdataset_validation.repeat()
else:
self.train_data = self.images
self.tfdataset_train = tfdataset.skip(
math.ceil(self.validation_split * self.dataset_size))
self.tfdataset_train = self.tfdataset_train.map(self._parse_function)
self.tfdataset_train = self.tfdataset_train.batch(self.batch_size)
self.tfdataset_train = self.tfdataset_train.prefetch(
tf.contrib.data.AUTOTUNE)
self.tfdataset_train = self.tfdataset_train.repeat()