self.grayscale = grayscale
def __len__(self):
return int(np.ceil(len(self.filenames) / float(self.batch_size)))
def __getitem__(self, idx):
batch_x = self.filenames[idx * self.batch_size:(idx + 1) * self.batch_size]
batch_x = np.array([read_img(filename, self.grayscale) for filename in batch_x])
batch_x = batch_x / 255.
return batch_x, batch_x
# data
if cfg.aug_dir and cfg.do_aug:
img_list = generate_image_list(cfg)
augment_images(img_list, cfg)
dataset_dir = cfg.aug_dir if cfg.aug_dir else cfg.train_data_dir
file_list = glob(dataset_dir + '/*')
num_valid_data = int(np.ceil(len(file_list) * 0.2))
data_train = data_flow(file_list[:-num_valid_data], cfg.batch_size, cfg.grayscale)
data_valid = data_flow(file_list[-num_valid_data:], cfg.batch_size, cfg.grayscale)
# loss
if cfg.loss == 'ssim_loss':
@tf.function
def ssim_loss(gt, y_pred, max_val=1.0):
return 1 - tf.reduce_mean(tf.image.ssim(gt, y_pred, max_val=max_val))
loss = ssim_loss
def main():
MAX = None
MODEL_NAME = 'SHALLOW'
NB_ROWS, NB_COLS = 150, 150
BATCH_SIZE = 30
NB_TEST_PATCHES = 50
images, filenames = [], []
for fn in glob.glob('../data/ICDAR_CLAMM_task_1_3/*.tif'):
img = np.array(io.imread(fn, as_grey=True), dtype='float32')
scaled = img / np.float32(255.0)
scaled = downscale_local_mean(scaled, factors=(2, 2))
images.append(scaled)
filenames.append(os.path.basename(fn))
if MAX and len(images) >= MAX:
break
script_encoder = \
pickle.load(open('models/' +
MODEL_NAME + '/script_encoder.p', 'rb'))
date_encoder = \
pickle.load(open('models/' +
MODEL_NAME + '/date_encoder.p', 'rb'))
model = model_from_json(
open('models/' + MODEL_NAME + '/architecture.json').read())
model.compile(loss='categorical_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
model.load_weights('models/' + MODEL_NAME + '/weights.hdf5')
script_probas, date_probas = [], []
for image, filename in zip(images, filenames):
print(filename)
X, _, _ = utils.augment_images(images=[image],
scripts=[1],
dates=[1],
nb_rows=NB_ROWS,
nb_cols=NB_COLS,
nb_patches=NB_TEST_PATCHES,
distort=False)
pred_scr_Y, pred_date_Y = \
model.predict({'img_input': X},
batch_size=BATCH_SIZE)
pred_scr = np.array(pred_scr_Y)
pred_date = np.array(pred_date_Y)
av_pred_scr = pred_scr.mean(axis=0)
av_pred_date = pred_date.mean(axis=0)
script_probas.append(av_pred_scr)
date_probas.append(av_pred_date)
# Task 1: script for clean data (13 cols)
labels = [str(i) for i in range(1, 13)]
idxs = [list(script_encoder.classes_).index(l) for l in labels]
with open('../output/task1_clean_script_belonging.txt', 'w') as f:
for fn, probas in zip(filenames, script_probas):
f.write(fn + ', ')
scores = [str(probas[i]) for i in idxs]
f.write(', '.join(scores))
f.write('\n')
# Task 3: date for clean data (13 cols)
labels = [str(i) for i in range(1, 16)]
idxs = [list(date_encoder.classes_).index(l) for l in labels]
with open('../output/task3_clean_date_belonging.txt', 'w') as f:
for fn, probas in zip(filenames, date_probas):
f.write(fn + ', ')
scores = [str(probas[i]) for i in idxs]
f.write(', '.join(scores))
f.write('\n')
# Task 1: distances for clean data (13 cols)
X = np.asarray(script_probas)
dm = squareform(pdist(X, minmax))
dm = dm / dm.sum(axis=1)[:, np.newaxis]
with open('../output/task1_clean_script_distance.txt', 'w') as f:
f.write('L/C ,')
f.write(', '.join(filenames) + '\n')
for n, v in zip(filenames, dm):
f.write(n + ', ')
f.write(', '.join([str(sc) for sc in v]))
f.write('\n')
# Task 3: distances for clean data (13 cols)
X = np.asarray(date_probas)
dm = squareform(pdist(X, minmax))
dm = dm / dm.sum(axis=1)[:, np.newaxis]
with open('../output/task3_clean_date_distance.txt', 'w') as f:
f.write('L/C ,')
f.write(', '.join(filenames) + '\n')
for n, v in zip(filenames, dm):
f.write(n + ', ')
f.write(', '.join([str(sc) for sc in v]))
f.write('\n')
def train():
#l = {'4', '8'}
l = None
train_images, train_scripts, train_dates = \
utils.load_dir('../data/splits/train',
max_nb=None,
script_classes=l)
dev_images, dev_scripts, dev_dates = \
utils.load_dir('../data/splits/dev',
max_nb=None,
script_classes=l)
try:
os.mkdir('models')
except:
pass
try:
shutil.rmtree('models/' + MODEL_NAME)
except:
pass
os.mkdir('models/' + MODEL_NAME)
script_encoder = LabelEncoder().fit(train_scripts)
date_encoder = LabelEncoder().fit(train_dates)
orig_train_script_int = script_encoder.transform(train_scripts)
orig_train_date_int = date_encoder.transform(train_dates)
orig_dev_script_int = script_encoder.transform(dev_scripts)
orig_dev_date_int = date_encoder.transform(dev_dates)
nb_scripts = len(script_encoder.classes_)
nb_dates = len(date_encoder.classes_)
print('-> Working on', nb_scripts, 'script classes:',
script_encoder.classes_)
print('-> Working on', nb_dates, 'date classes:', date_encoder.classes_)
pickle.dump(script_encoder,
open('models/' + MODEL_NAME + '/script_encoder.p', 'wb'))
pickle.dump(date_encoder,
open('models/' + MODEL_NAME + '/date_encoder.p', 'wb'))
model = ShallowNet(nb_scripts=nb_scripts,
nb_dates=nb_dates,
nb_rows=NB_ROWS,
nb_cols=NB_COLS)
model.summary()
with open('models/' + MODEL_NAME + '/architecture.json', 'w') as F:
F.write(model.to_json())
best_dev_acc = 0.0
print('-> cropping dev...')
dev_crops = []
for image in dev_images:
Xs, _, _ = utils.augment_images(images=[image],
scripts=[1],
dates=[1],
nb_rows=NB_ROWS,
nb_cols=NB_COLS,
nb_patches=NB_TEST_PATCHES,
distort=False)
dev_crops.append(Xs)
example = dev_crops[0]
for idx, crop in enumerate(example):
crop = crop.reshape((crop.shape[1], crop.shape[2]))
imsave(str(idx) + '_crop.png', crop)
for e in range(NB_EPOCHS):
print('-> in epoch', e)
print('-> cropping train...')
train_X, train_script_int, train_date_int = \
utils.augment_images(images=train_images,
scripts=orig_train_script_int,
dates=orig_train_date_int,
nb_rows=NB_ROWS,
nb_cols=NB_COLS,
nb_patches=NB_TRAIN_PATCHES,
distort=True)
train_scr_Y = np_utils.to_categorical(train_script_int,
num_classes=nb_scripts)
train_date_Y = np_utils.to_categorical(train_date_int,
num_classes=nb_dates)
model.fit({'img_input': train_X}, {
'script_out': train_scr_Y,
'date_out': train_date_Y
},
batch_size=BATCH_SIZE,
epochs=1,
shuffle=True)
script_probas = []
date_probas = []
patch_script_gold = []
patch_date_gold = []
patch_script_pred = []
patch_date_pred = []
for dev_x, o_scr, o_date in zip(dev_crops, orig_dev_script_int,
orig_dev_date_int):
pred_scr_Y, pred_date_Y = \
model.predict({'img_input': dev_x},
batch_size=BATCH_SIZE)
patch_script_pred.extend(list(np.argmax(pred_scr_Y, axis=-1)))
patch_date_pred.extend(list(np.argmax(pred_date_Y, axis=-1)))
patch_script_gold.extend([o_scr] * NB_TEST_PATCHES)
patch_date_gold.extend([o_date] * NB_TEST_PATCHES)
av_pred_scr = pred_scr_Y.mean(axis=0)
av_pred_date = pred_date_Y.mean(axis=0)
script_probas.append(av_pred_scr)
date_probas.append(av_pred_date)
dev_script_pred = np.argmax(script_probas, axis=-1)
dev_date_pred = np.argmax(date_probas, axis=-1)
# patch level:
curr_patch_script_acc = accuracy_score(patch_script_pred,
patch_script_gold)
print(' curr script val acc (patch level):', curr_patch_script_acc)
curr_patch_date_acc = accuracy_score(patch_date_pred, patch_date_gold)
print(' curr date val acc: (patch level)', curr_patch_date_acc)
# image level:
curr_script_acc = accuracy_score(dev_script_pred, orig_dev_script_int)
print(' curr script val acc (image level):', curr_script_acc)
curr_date_acc = accuracy_score(dev_date_pred, orig_dev_date_int)
print(' curr date val acc: (image level)', curr_date_acc)
curr_acc = (curr_patch_script_acc + curr_patch_date_acc) / 2.0
# save weights, if appropriate:
if curr_acc > best_dev_acc:
print(' -> saving model')
model.save_weights('models/' + MODEL_NAME + '/weights.hdf5',
overwrite=True)
best_dev_acc = curr_acc
# half learning rate:
if e and e % 20 == 0:
old_lr = model.optimizer.lr.get_value()
new_lr = np.float32(old_lr * 0.1)
model.optimizer.lr.set_value(new_lr)
print('\t- Lowering learning rate > was:', old_lr, ', now:',
new_lr)
train_mean = np.mean(synthetic_images)
train_std = np.std(synthetic_images)
mean_std_path = '%s/meanstd.npz' % (expdir)
np.savez(mean_std_path, train_mean=train_mean, train_std=train_std)
# standardize simulated data for input to refiner
synthetic_images = (synthetic_images - train_mean) / train_std
# normalize real images to be between -1 and 1 for discriminator
#MAXVAL = 2**14 - 1
max_val = np.amax(real_images, axis=(1, 2), keepdims=True)
real_images = (real_images / max_val) - np.mean(
real_images / max_val, axis=(1, 2), keepdims=True)
if args.augment: # generate more training data through augmentation
synthetic_images = utils.augment_images(synthetic_images)
real_images = utils.augment_images(real_images)
#synth_crops = utils.random_crop_generator(synthetic_images, args.crop_size, args.batch_size)
#real_crops = utils.random_crop_generator(real_images, args.crop_size, args.batch_size)
#synth_crops = utils.center_crop_generator(synthetic_images, args.crop_size, args.batch_size)
#real_crops = utils.center_crop_generator(real_images, args.crop_size, args.batch_size)
synth_crops = utils.random_generator(synthetic_images, args.batch_size)
real_crops = utils.random_generator(real_images, args.batch_size)
# select optimizer to use during training
if args.optimizer.lower() == 'sgd':
optimizer = SGD(args.lr, momentum=0.9)
elif args.optimizer.lower() == 'adam':
optimizer = Adam(args.lr, amsgrad=True)
else:
def test():
#l = {'4', '8'}
l = None
train_images, train_scripts, train_dates = \
utils.load_dir('../data/splits/train',
max_nb=None,
script_classes=l)
dev_images, dev_scripts, dev_dates = \
utils.load_dir('../data/splits/dev',
max_nb=None,
script_classes=l)
script_encoder = \
pickle.load(open('models/' +
MODEL_NAME + '/script_encoder.p', 'rb'))
date_encoder = \
pickle.load(open('models/' +
MODEL_NAME + '/date_encoder.p', 'rb'))
model = model_from_json(
open('models/' + MODEL_NAME + '/architecture.json').read())
model.compile(loss='categorical_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
model.load_weights('models/' + MODEL_NAME + '/weights.hdf5')
orig_train_script_int = script_encoder.transform(train_scripts)
orig_train_date_int = date_encoder.transform(train_dates)
orig_dev_script_int = script_encoder.transform(dev_scripts)
orig_dev_date_int = date_encoder.transform(dev_dates)
print('-> cropping dev...')
dev_crops = []
for image in dev_images:
Xs, _, _ = utils.augment_images(images=[image],
scripts=[1],
dates=[1],
nb_rows=NB_ROWS,
nb_cols=NB_COLS,
nb_patches=NB_TEST_PATCHES,
distort=False)
dev_crops.append(Xs)
script_probas = []
date_probas = []
patch_script_gold = []
patch_date_gold = []
patch_script_pred = []
patch_date_pred = []
for dev_x, o_scr, o_date in zip(dev_crops, orig_dev_script_int,
orig_dev_date_int):
pred_scr_Y, pred_date_Y = \
model.predict({'img_input': dev_x},
batch_size=BATCH_SIZE)
patch_script_pred.extend(list(np.argmax(pred_scr_Y, axis=-1)))
patch_date_pred.extend(list(np.argmax(pred_date_Y, axis=-1)))
patch_script_gold.extend([o_scr] * NB_TEST_PATCHES)
patch_date_gold.extend([o_date] * NB_TEST_PATCHES)
av_pred_scr = pred_scr_Y.mean(axis=0)
av_pred_date = pred_date_Y.mean(axis=0)
script_probas.append(av_pred_scr)
date_probas.append(av_pred_date)
dev_script_pred = np.argmax(script_probas, axis=-1)
dev_date_pred = np.argmax(date_probas, axis=-1)
# patch level:
curr_patch_script_acc = accuracy_score(patch_script_pred,
patch_script_gold)
print(' curr script val acc (patch level):', curr_patch_script_acc)
curr_patch_date_acc = accuracy_score(patch_date_pred, patch_date_gold)
print(' curr date val acc: (patch level)', curr_patch_date_acc)
# image level:
curr_script_acc = accuracy_score(dev_script_pred, orig_dev_script_int)
print(' curr script val acc (image level):', curr_script_acc)
curr_date_acc = accuracy_score(dev_date_pred, orig_dev_date_int)
print(' curr date val acc: (image level)', curr_date_acc)