sys.stdout.flush()
mean_file = os.path.join(configuration.get_data_dir(), "mean.dat")
shape_file = os.path.join(configuration.get_data_dir(), "shape.dat")
#
input_shape = np.fromfile(shape_file, dtype=np.int32)
mean_image = np.fromfile(mean_file, dtype=np.float32)
mean_image = np.reshape(mean_image, input_shape)
number_of_classes = configuration.get_number_of_classes()
# loading tfrecords into a dataset object
if pargs.mode == "train":
tr_dataset = tf.data.TFRecordDataset(tfr_train_file)
tr_dataset = tr_dataset.map(
lambda x: data.parser_tfrecord(x,
input_shape,
mean_image,
number_of_classes,
with_augmentation=True))
tr_dataset = tr_dataset.shuffle(configuration.get_shuffle_size())
tr_dataset = tr_dataset.batch(
batch_size=configuration.get_batch_size())
if pargs.mode == "train" or pargs.mode == "test":
val_dataset = tf.data.TFRecordDataset(tfr_test_file)
val_dataset = val_dataset.map(
lambda x: data.parser_tfrecord(x,
input_shape,
mean_image,
number_of_classes,
with_augmentation=False))
val_dataset = val_dataset.batch(
configuration = conf.ConfigurationFile(configuration_file, pargs.name)
#parser_tf_record
#/home/vision/smb-datasets/MNIST-5000/ConvNet2.0/
tfr_train_file = os.path.join(configuration.get_data_dir(), pargs.file)
mean_file = os.path.join(configuration.get_data_dir(), "mean.dat")
shape_file = os.path.join(configuration.get_data_dir(), "shape.dat")
#
input_shape = np.fromfile(shape_file, dtype=np.int32)
mean_image = np.fromfile(mean_file, dtype=np.float32)
mean_image = np.reshape(mean_image, input_shape)
number_of_classes = configuration.get_number_of_classes()
tr_dataset = tf.data.TFRecordDataset(tfr_train_file)
tr_dataset = tr_dataset.map(lambda x: data.parser_tfrecord(
x, input_shape, 0, number_of_classes, False))
tr_dataset = tr_dataset.shuffle(configuration.get_shuffle_size())
tr_dataset = tr_dataset.batch(batch_size=configuration.get_batch_size())
n_rows = 8
n_cols = 8
fig, xs = plt.subplots(n_rows, n_cols)
for i in range(n_rows):
for j in range(n_cols):
xs[i, j].set_axis_off()
for image, label in tr_dataset:
for i in range(64):
row = int(i / n_cols)
col = i % n_cols
im = image[i]
#im = 255 * (image[i] - np.min(image[i]))/ (np.max(image[i])-np.min(image[i]))
tfr_train_file=[os.path.join(configuration.get_data_dir(), "train_{}.tfrecords".format(idx)) for idx in range(configuration.get_num_threads())]
if pargs.mode == 'train' or pargs.mode == 'test':
tfr_test_file=[os.path.join(configuration.get_data_dir(), "test_{}.tfrecords".format(idx)) for idx in range(configuration.get_num_threads())]
sys.stdout.flush()
mean_file = os.path.join(configuration.get_data_dir(), "mean.dat")
shape_file = os.path.join(configuration.get_data_dir(),"shape.dat")
#
input_shape = np.fromfile(shape_file, dtype=np.int32)
mean_image = np.fromfile(mean_file, dtype=np.float32)
mean_image = np.reshape(mean_image, input_shape)
number_of_classes = configuration.get_number_of_classes()
#loading tfrecords into a dataset object
if pargs.mode == 'train' :
tr_dataset = tf.data.TFRecordDataset(tfr_train_file)
tr_dataset = tr_dataset.map(lambda x : data.parser_tfrecord(x, input_shape, mean_image, number_of_classes, with_augmentation = True));
tr_dataset = tr_dataset.shuffle(configuration.get_shuffle_size())
tr_dataset = tr_dataset.batch(batch_size = configuration.get_batch_size())
if pargs.mode == 'train' or pargs.mode == 'test':
val_dataset = tf.data.TFRecordDataset(tfr_test_file)
val_dataset = val_dataset.map(lambda x : data.parser_tfrecord(x, input_shape, mean_image, number_of_classes, with_augmentation = False));
val_dataset = val_dataset.batch(batch_size = configuration.get_batch_size())
#Defining callback for saving checkpoints
#save_freq: frequency in terms of number steps each time checkpoint is saved
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=os.path.join(configuration.get_snapshot_dir(), '{epoch:03d}.h5'),
save_weights_only=True,
mode = 'max',
configuration_file = pargs.config
configuration = conf.ConfigurationFile(configuration_file, pargs.name)
#parser_tf_record
#/home/vision/smb-datasets/MNIST-5000/CC7221-HW2.0/
tfr_train_file = os.path.join(configuration.get_data_dir(), pargs.file)
mean_file = os.path.join(configuration.get_data_dir(), "mean.dat")
shape_file = os.path.join(configuration.get_data_dir(), "shape.dat")
#
input_shape = np.fromfile(shape_file, dtype=np.int32)
mean_image = np.fromfile(mean_file, dtype=np.float32)
mean_image = np.reshape(mean_image, input_shape)
number_of_classes = configuration.get_number_of_classes()
tr_dataset = tf.data.TFRecordDataset(tfr_train_file)
tr_dataset = tr_dataset.map(lambda x: data.parser_tfrecord(
x, input_shape, mean_image, number_of_classes, 'test'))
tr_dataset = tr_dataset.shuffle(configuration.get_shuffle_size())
tr_dataset = tr_dataset.batch(batch_size=configuration.get_batch_size())
fig, xs = plt.subplots(2, 5)
for image, label in tr_dataset:
for i in range(10):
row = int(i / 5)
col = i % 5
im = 255 * (image[i] - np.min(image[i])) / (np.max(image[i]) -
np.min(image[i]))
xs[row, col].imshow(np.uint8(im), cmap='gray')
plt.pause(1)