def download(self, links, target_folder='./data'):
"""Download images from a lisk of links"""
# check links and folder:
if len(links) < 1:
print("Error: Empty list, no links provided")
exit()
self.images_links = links
DatasetBuilder.check_folder_existance(target_folder)
if target_folder[-1] == '/':
target_folder = target_folder[:-1]
# start downloading:
print("Downloading files...")
progress = 0
images_nbr = sum(
[len(self.images_links[key]) for key in self.images_links])
for keyword, links in self.images_links.items():
DatasetBuilder.check_folder_existance(target_folder + '/' +
keyword,
display_msg=False)
for link in links:
target_file = target_folder + '/' + keyword + '/' + link.split(
'/')[-1]
try:
f = urllib.URLopener()
f.retrieve(link, target_file)
except IOError:
self.failed_links.append(link)
progress = progress + 1
print("\r >> Download progress: %d/%d" %
(progress, images_nbr),
end="")
sys.stdout.flush()
print("\r >> Download progress: ", (progress * 100 / images_nbr), "%")
print(" >> ", (progress - len(self.failed_links)),
" images downloaded")
# save failed links:
if len(self.failed_links):
f2 = open(target_folder + "/failed_list.txt", 'w')
for link in self.failed_links:
f2.write(link + "\n")
print(" >> Failed to download ", len(self.failed_links),
" images: access not granted ", "(links saved to: '",
target_folder, "/failed_list.txt')")
def createRealDatasets(dir='topics/'):
'''
@return: A dictionary of the real datasets, with their names as keys.
'''
builder = DatasetBuilder()
datasets = {}
datasets['Windows 7: Features or speed?'] = builder.fromFiles(
[dir + 'features_windows7.txt.data', dir + 'speed_windows7.txt.data'])
datasets['Best Western Hotels: Bath or room?'] = builder.fromFiles([
dir + 'bathroom_bestwestern_hotel_sfo.txt.data',
dir + 'rooms_bestwestern_hotel_sfo.txt.data'
])
datasets['Hotels: San Francisco or Chicago?'] = builder.fromFiles([
dir + 'rooms_bestwestern_hotel_sfo.txt.data',
dir + 'rooms_swissotel_chicago.txt.data'
])
datasets['Battery life: Which device?'] = builder.fromFiles([
dir + 'battery-life_amazon_kindle.txt.data',
dir + 'battery-life_ipod_nano_8gb.txt.data',
dir + 'battery-life_netbook_1005ha.txt.data'
])
datasets['iPod Nano: Which aspect?'] = builder.fromFiles([
dir + 'battery-life_ipod_nano_8gb.txt.data',
dir + 'screen_ipod_nano_8gb.txt.data',
dir + 'sound_ipod_nano_8gb.txt.data',
dir + 'video_ipod_nano_8gb.txt.data'
])
return datasets
def config_path(self, config_path):
"""Load config file (.ini file) and get dataset builder and neural
network model. Set private attribute self._dataset_builder, self._model
and self._config_path. Note that this function does not parse config
file. It simply uses config file to set up dataset builder and model
builder.
Args:
config_path: Str, path to config (.ini) file.
"""
if not os.path.isfile(config_path):
raise ValueError("Specified config file does not exist.")
# Get dataset builder based on config file
self._dataset_builder = DatasetBuilder(config_path=config_path,
one_hot=False)
# Get model builder and get encoder (triplet model with weights)
self._model_builder = ModelBuilder(config_path=config_path)
self._model = self._model_builder.get_encoder()
# Set self._config_path
self._config_path = config_path
def createRealDatasets(dir = 'topics/'):
'''
@return: A dictionary of the real datasets, with their names as keys.
'''
builder = DatasetBuilder()
datasets = {}
datasets['Windows 7: Features or speed?'] = builder.fromFiles([dir+'features_windows7.txt.data', dir+'speed_windows7.txt.data'])
datasets['Best Western Hotels: Bath or room?'] = builder.fromFiles([dir+'bathroom_bestwestern_hotel_sfo.txt.data', dir+'rooms_bestwestern_hotel_sfo.txt.data'])
datasets['Hotels: San Francisco or Chicago?'] = builder.fromFiles([dir+'rooms_bestwestern_hotel_sfo.txt.data', dir+'rooms_swissotel_chicago.txt.data'])
datasets['Battery life: Which device?'] = builder.fromFiles([dir+'battery-life_amazon_kindle.txt.data', dir+'battery-life_ipod_nano_8gb.txt.data', dir+'battery-life_netbook_1005ha.txt.data'])
datasets['iPod Nano: Which aspect?'] = builder.fromFiles([dir+'battery-life_ipod_nano_8gb.txt.data', dir+'screen_ipod_nano_8gb.txt.data', dir+'sound_ipod_nano_8gb.txt.data', dir+'video_ipod_nano_8gb.txt.data'])
return datasets
type=int,
default=None)
parser.add_argument('--crop_w',
action='store',
help='width of the cropped center',
type=int,
default=None)
parser.add_argument('--chessboard_h',
type=int,
default=7,
help='chessboard height')
parser.add_argument('--chessboard_w',
type=int,
default=7,
help='chessboard width')
parser.add_argument('--name',
action='store',
type=str,
help='dataset name')
parser.add_argument('--flip', action='store_true')
parser.add_argument('--fps_ratio', action='store', default=1)
args = parser.parse_args()
roi = (args.crop_x, args.crop_y, args.crop_h, args.crop_w)
if (not roi[0]) or (not roi[1]) or (not roi[2]) or (not roi[3]):
roi = None
DatasetBuilder(args.name, roi, (args.chessboard_h, args.chessboard_w),
args.flip, args.fps_ratio).build()
class RepresentationGenerator:
"""A representation (embeddings) generator for visualization of characters.
When embeddings and labels are written to files, embeddings are written to
'out_file'_vec.tsv file and labels are written to 'out_file'_meta.tsv file.
You can see a 3D visualization of the embeddings in browser using TensorFlow
embedding projector. Go to https://projector.tensorflow.org/ and click
'Load' button on the left-hand side. Load 'out_file'_vec.tsv as vectors and
'out_file'_meta.tsv as metadata to visualize embeddings.
Initialization:
>>> rg = RepresentationGenerator(config_path='configs/config.ini', \
out_dir='embeddings/')
Configurations are all set in .ini file. Change path to new config file to
change configurations:
>>> rg.config_path = 'configs/new_config.ini'
Get representations for all images in a directory (IMPORTANT! Expect all
images to be generated by VisualizationGenerator in vis_gen. Otherwise, make
sure file name in format 'U+XXXX_*' to use 'char_as_label' feature):
>>> codepoints, embeddings = rg.get_embeddings(img_dir='test_imgs', \
char_as_label=True)
Write codepoints and embeddings to file:
>>> rg.write_embeddings_from_list(codepoints=codepoints, \
embeddings=embeddings)
Write write labels and embeddings to file directly from image directory.
>>> rg.write_embeddings_from_image(img_dir='test_imgs')
"""
def __init__(self,
config_path='configs/sample_config.ini',
out_dir="embeddings"):
"""Need a checkpoint directory to initialize RepresentationGenerator.
Args:
config_path: Str, path to config (.ini) file. (default
"configs/sample_config.ini"
out_dir: Str, relative path of the output directory (default
"embeddings").
Raises:
ValueError: if model_name not found
ValueError: if ckpt_dir don't contain TensorFlow formatted
checkpoint
"""
self._dataset_builder = None
self._model_builder = None
self._model = None
self.config_path = config_path
self.out_dir = out_dir
@property
def config_path(self):
"""
Returns:
self._config_path: Str, path to config file (.ini file).
"""
return self._config_path
@property
def out_dir(self):
"""
Returns:
self._out_dir: Str, path to output directory.
"""
return self._out_dir
@config_path.setter
def config_path(self, config_path):
"""Load config file (.ini file) and get dataset builder and neural
network model. Set private attribute self._dataset_builder, self._model
and self._config_path. Note that this function does not parse config
file. It simply uses config file to set up dataset builder and model
builder.
Args:
config_path: Str, path to config (.ini) file.
"""
if not os.path.isfile(config_path):
raise ValueError("Specified config file does not exist.")
# Get dataset builder based on config file
self._dataset_builder = DatasetBuilder(config_path=config_path,
one_hot=False)
# Get model builder and get encoder (triplet model with weights)
self._model_builder = ModelBuilder(config_path=config_path)
self._model = self._model_builder.get_encoder()
# Set self._config_path
self._config_path = config_path
@out_dir.setter
def out_dir(self, out_dir):
"""
Args:
out_dir: Str, relative path of the output directory.
"""
self._out_dir = out_dir
def get_embeddings(self, img_dir):
"""For the image files in 'img_dir', return their embeddings.
Args:
img_dir: Str, relative path to directory where all character images
are stored.
Returns:
codepoints: List of codepoints with other configs in format
'CODEPOINT_FONTNAME[_FONTSTYLE]_ANTIALIAS'. Same as the filename
generated by vis_gen.
embeddings: List of embeddings. Each element is a representation of
a character.
"""
# Get dataset with filename as label
dataset = self._dataset_builder.get_filename_dataset(img_dir)
# Get unicode code points and their corresponding embeddings
codepoints = []
embeddings = []
i = 0
print('Generating embeddings...')
for img, filename in dataset:
i += 1
if i % 100 == 0:
print("Getting embedding #" + str(i) + ".")
# decode Tensor into string
filename_str = filename.numpy()[0].decode('utf-8')
codepoints.append(filename_str.split('.')[0])
# Get embeddings
embedding = self._model.predict(img)[0]
embeddings.append(embedding)
return codepoints, embeddings
def write_embeddings_from_image(self,
img_dir,
out_file,
char_as_label=True):
"""Get embeddings and write embeddings and labels to .tsv files. This
function will write to two .tsv files: 'out_file'_vec.tsv and
'out_file'_meta.tsv. Entries in 'out_file'_vec.tsv are separated by
newline. Elements in each embeddings are separated by tab. Entries in
'out_file'_meta.tsv are separated by newline.
Args:
img_dir: Str, relative path to directory where all character images
are stored
out_file: Str, name of the output file intended to write to
char_as_label: Bool, whether to
"""
# Get model predictions and unicode code points
codepoints, embeddings = self.get_embeddings(img_dir=img_dir)
# Write code points to '_meta.tsv' and embeddgins to '_vec.tsv'
self.write_embeddings_from_list(codepoints, embeddings, out_file,
char_as_label)
def write_embeddings_from_list(self,
codepoints,
embeddings,
out_file,
char_as_label=True):
"""Write labels and embeddings to file.
Args:
codepoints: List of Str, each element must be in format 'U+XXXX'.
embeddings:
out_file: Str, name of the output file intended to write to.
char_as_label: Bool, whether to use character as label. Otherwise,
use code points.
Raises:
ValueError: if codepoints and embeddings does not have the same
number of entries.
"""
# Throw exception if codepoint array and embedding array does not have
# the same number of elements
if len(codepoints) != len(embeddings):
raise ValueError('Expect array codepoints and embeddings to have '
'the same number of elements.')
# Get absolute directory path, create new folder if needed.
out_dir_abs = os.path.abspath(self.out_dir)
os.makedirs(out_dir_abs, exist_ok=True)
# Get absolute path to output files
out_file_abs = os.path.join(out_dir_abs, out_file)
out_file_vec_abs = out_file_abs + '_vec.tsv'
out_file_meta_abs = out_file_abs + '_meta.tsv'
# Write embeddings to file
print("Writing embeddings to file {}...".format(out_file_vec_abs))
np.savetxt(out_file_abs + "_vec.tsv", embeddings, delimiter='\t')
print('Successfully written to file {}.'.format(out_file_vec_abs))
# Change Unicode code point to character if specified
if char_as_label:
try:
# 'CODEPOINT_FONTNAME[_FONTSTYLE]_ANTIALIAS' -> 'CODEPOINT'
codepoints = [
codepoint.split('_')[0] for codepoint in codepoints
]
# 'U+XXXX' -> char
codepoints = [
chr(int('0x' + codepoint[2:], 16))
for codepoint in codepoints
]
except:
print('All entries of codepoints array must be in format: '
'CODEPOINT_FONTNAME[_FONTSTYLE]_ANTIALIAS. Example: '
'U+4eba_Noto Sans CJK SC_Default.')
raise
# Write labels
print("Writing labels to file {}...".format(out_file_meta_abs))
with open(out_file_meta_abs, "w+") as f_out:
for label in codepoints:
f_out.write(label)
f_out.write('\n')
print('Successfully written to file {}.'.format(out_file_meta_abs))
def run(feature_names, wordemb_key, with_nonnatural, emb_dim, activation, lr,
epochs, batch_size, criterion, id):
# prepare data
dataset = DatasetBuilder(data,
feature_names=feature_names,
wordemb_key=wordemb_key,
with_nonnatural=with_nonnatural)
train_loader, validation_loader = dataset_split(dataset,
validation_split=0.2,
batch_size=batch_size,
seed=42)
# set hyperparameters
activation = activation
emb_dim = emb_dim
lr = lr
input_size = dataset[0][1].shape[0]
output_size = len(data["experts"])
if criterion == "BCELoss":
criterion = torch.nn.BCELoss()
elif criterion == "BCEWithLogitsLoss":
criterion = torch.nn.BCEWithLogitsLoss()
# create network matrices
W1 = Variable(torch.randn(emb_dim, input_size).float(), requires_grad=True)
W2 = Variable(torch.randn(output_size, emb_dim).float(),
requires_grad=True)
# monitoring variable
best_accuracy = 0
# run model
for epoch in range(epochs):
# model monitoring variables
loss_val = 0
win = 0
lose = 0
for expert_id, input_emb, target in train_loader:
# transform input data
x = Variable(input_emb).float().transpose(0, 1) #.to(device)
y_true = Variable(target, 0).float() #.to(device)
expert_id = Variable(torch.unsqueeze(expert_id,
0).long()) #.to(device)
# run matrix multiplication
z1 = torch.matmul(W1, x)
a1 = torch.sigmoid(z1)
z2 = torch.matmul(W2, a1)
# run output activation
if activation == 'sigmoid':
a2 = torch.sigmoid(z2)
elif activation == 'softmax':
a2 = torch.softmax(z2, 0)
else:
a2 = z2
# create loss and accuracy measures (if batch_size > 1 the output needs to be transformed)
if batch_size > 1:
a3 = torch.zeros((a2[0].shape[0], 1))
for idx, eid in enumerate(expert_id[0]):
a3[idx, 0] = a2[eid, idx]
loss = criterion(a3, y_true)
else:
loss = criterion(a2[expert_id][0], y_true)
# backpropagation
loss.backward()
loss_val += loss
W1.data -= lr * W1.grad.data
W2.data -= lr * W2.grad.data
# calculate accuracy measures
if batch_size > 1:
for result in torch.abs(a3 - y_true) < 0.5:
if result == 0:
lose += 1
else:
win += 1
else:
if torch.abs(a2[expert_id][0][0][0] - y_true[0][0]) < 0.5:
win += 1
else:
lose += 1
# zero gradients
W1.grad.data.zero_()
W2.grad.data.zero_()
ex.log_scalar("train_accuracies", win / (win + lose))
ex.log_scalar("train_loss", int(loss_val))
# deactivate gradients to store test loss information for every epoch
with torch.no_grad():
# model monitoring variables
win = 0
lose = 0
# run model over validation dataset
for expert_id, input_emb, target in validation_loader:
# transform input data
x = Variable(input_emb).float().transpose(0, 1) #.to(device)
y_true = Variable(target, 0).float() #.to(device)
expert_id = Variable(torch.unsqueeze(expert_id,
0).long()) #.to(device)
# run matrix multiplication
z1 = torch.matmul(W1, x)
a1 = torch.sigmoid(z1)
z2 = torch.matmul(W2, a1)
# run output activation
if activation == 'sigmoid':
a2 = torch.sigmoid(z2)
elif activation == 'softmax':
a2 = torch.softmax(z2, 0)
else:
a2 = z2
# calculate accuracy measures
if batch_size > 1:
a3 = torch.zeros((a2[0].shape[0], 1))
for idx, eid in enumerate(expert_id[0]):
a3[idx, 0] = a2[eid, idx]
for result in torch.abs(a3 - y_true) < 0.5:
if result == 0:
lose += 1
else:
win += 1
else:
if torch.abs(a2[expert_id][0][0][0] - y_true[0][0]) < 0.5:
win += 1
else:
lose += 1
# log result
ex.log_scalar("val_accuracies", win / (win + lose))
if win / (win + lose) > best_accuracy:
best_accuracy = win / (win + lose)
if epoch % 5 == 0:
print(f'Loss at epo {epoch}: {int(loss_val)/len(dataset)}')
# save model
torch.save(
W1, './models/softmax_classifier/softmax_classifier_W1_' + str(id) +
'.tar')
torch.save(
W2, './models/softmax_classifier/softmax_classifier_W2_' + str(id) +
'.tar')
return best_accuracy
def run(feature_names, query, wordemb_key, expemb_size, lin_layer_sizes,
expemb_dropout, lin_layer_dropouts, lr, epochs, batch_size, id):
device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
# TRAINING
# build dataset, train/test split and data loaders
dataset = DatasetBuilder(data,
feature_names=feature_names,
query=query,
wordemb_key=wordemb_key)
train_loader, validation_loader = dataset_split(dataset,
validation_split=0.2,
batch_size=batch_size,
seed=42)
# create model with hyperparameters and set criterion
model = BinaryClassifier(
(dataset.expert_id_length, expemb_size),
dataset.input_size,
lin_layer_sizes=lin_layer_sizes,
output_size=1,
emb_dropout=expemb_dropout,
lin_layer_dropouts=lin_layer_dropouts).to(device).double()
criterion = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
train_step = make_train_step(model, criterion, optimizer)
accuracies = []
for epoch in range(epochs):
for emb_x, feat_x, y in train_loader:
emb_x = emb_x.to(device)
feat_x = feat_x.to(device)
y = y.to(device)
# Forward Pass
# run training step as explained in the helper function
preds, loss = train_step(emb_x, feat_x, y)
ex.log_scalar("train_losses", loss)
with torch.no_grad():
ex.log_scalar(
"train_accuracies",
accuracy(preds.cpu(),
torch.tensor(y, dtype=torch.long).cpu()))
# deactivate gradients to store test loss information for every epoch
with torch.no_grad():
for emb_x_val, feat_x_val, y_val in validation_loader:
emb_x_val = emb_x_val.to(device)
feat_x_val = feat_x_val.to(device)
y_val = y_val.to(device)
model.eval()
preds = model(emb_x_val, feat_x_val)
val_loss = criterion(preds, y_val)
ex.log_scalar("val_losses", val_loss.item())
accuracies.append(
accuracy(preds.cpu(),
torch.tensor(y_val, dtype=torch.long).cpu()))
ex.log_scalar(
"val_accuracies",
accuracy(preds.cpu(),
torch.tensor(y_val, dtype=torch.long).cpu()))
# save model
torch.save(model.state_dict(), './models/' + str(id) + '.tar')
# EVALUATION
'''datapoints = generate_datapoints(dataset)
dataset = DatasetBuilder(data, feature_names, datapoints = datapoints,query=query, wordemb_key=wordemb_key)
rankings = predict_all(dataset, model, device)
truth = Truth(f.document2expert, data, with_query=query, wordemb_key=wordemb_key, top_k = 10)
validator = Validator()
validator.validate(rankings, truth)
ex.log_scalar("mean_ndcg", np.mean(validator.ndcg))
ex.log_scalar("mean_precisionk", np.mean(validator.precisionk))'''
return np.mean(accuracies[-batch_size:])
# 2. (alernative to the previous line) Load URLs from a file instead of the web:
#crawler.load_urls(download_folder + "/links.txt")
#crawler.load_urls_from_json(download_folder + "/links.json")
# 3. Save URLs to download them later (optional):
crawler.save_urls(download_folder + "/links.txt")
#crawler.save_urls_to_json(download_folder + "/links.json")
# 4. Download the images:
crawler.download_images(target_folder=download_folder)
### Build the dataset ###
from dataset_builder import DatasetBuilder
dataset_builder = DatasetBuilder()
# 1. rename the downloaded images:
source_folder = download_folder
target_folder = download_folder + "_renamed"
dataset_builder.rename_files(source_folder, target_folder)
#dataset_builder.rename_files(source_folder, target_folder, extensions=('.jpg', '.jpeg', '.png', '.gif'))
# 2. Resize the images:
source_folder = download_folder
target_folder = download_folder + "_resized"
dataset_builder.reshape_images(source_folder, target_folder)
#dataset_builder.reshape_images(source_folder, target_folder, width=64, height=64)
# 3. Crop the images:
source_folder = download_folder
def __init__(self, config_path='configs/sample_config.ini'):
"""Read and set configuration from config file (.ini file) and create
keras.Model object or input function according to configuration. To add
new model, simply add new base model to self._MODEL_MAP.
Args:
config_path: Str, path to config (.ini) file.
Raises:
ValueError: if values in config file does not have the correct type.
ValueError: if optimizer does not exists in predefined map.
"""
# Pre-defined learning rate schedules
self._LR_SCHEDULE_MAP = {
'ExponentialDecay':
tf.keras.optimizers.schedules.ExponentialDecay,
'PiecewiseConstantDecay':
tf.keras.optimizers.schedules.PiecewiseConstantDecay,
'PolynomialDecay':
tf.keras.optimizers.schedules.PolynomialDecay,
}
# Pre-defined optimizers
self._OPTIMIZER_MAP = {
'Adam':
tf.keras.optimizers.Adam,
'RMSprop':
tf.keras.optimizers.RMSprop,
}
# Pre-defined losses
# IMPORTANT: DON'T USE TRIPLET HARD LOSS! EXTREMELY HARD TO TRAIN!
self._LOSS_MAP = {
'CrossEntropy':
tf.keras.losses.CategoricalCrossentropy,
'TripletHard':
tfa.losses.TripletHardLoss,
'TripletSemiHard':
tfa.losses.TripletSemiHardLoss,
}
# Pre-defined metrics
self._METRIC_MAP = {
'Accuracy':
tf.keras.metrics.CategoricalAccuracy,
}
# Get custom dataset
self.datset_builder = DatasetBuilder(config_path=config_path)
self.model_builder = ModelBuilder(config_path=config_path)
# Parse config file
config = configparser.ConfigParser()
config.read(config_path)
# Get classifier training config
self._CLS_CKPT_DIR = config.get('CLASSIFIER_TRAINING', 'CKPT_DIR')
self._CLS_MAX_STEP = config.getint('CLASSIFIER_TRAINING', 'MAX_STEP')
self._CLS_OPTIMIZER = config.get('CLASSIFIER_TRAINING', 'OPTIMIZER')
self._CLS_LR_BOUNDARIES = [
int(boundary.strip()) for boundary in
config.get('CLASSIFIER_TRAINING', 'LR_BOUNDARIES').split(',')
]
self._CLS_LR_VALUES = [
float(value.strip()) for value in
config.get('CLASSIFIER_TRAINING', 'LR_VALUES').split(',')
]
# Get triplet training config
self._TPL_INIT_DIR = config.get('TRIPLET_TRAINING', 'INIT_DIR')
self._TPL_CKPT_DIR = config.get('TRIPLET_TRAINING', 'CKPT_DIR')
self._TPL_CYCLES = config.getint('TRIPLET_TRAINING', 'CYCLES')
self._TPL_EPOCHS = config.getint('TRIPLET_TRAINING', 'EPOCHS')
self._TPL_FILTER_SIZE = config.getint('TRIPLET_TRAINING', 'FILTER_SIZE')
self._TPL_MARGIN = config.getfloat('TRIPLET_TRAINING', 'MARGIN')
self._TPL_OPTIMIZER = config.get('TRIPLET_TRAINING', 'OPTIMIZER')
self._TPL_LR_VALUE = config.getfloat('TRIPLET_TRAINING',
'LEARNING_RATE')
self._TPL_FREEZE_VARS = [
var.strip() for var in
config.get('TRIPLET_TRAINING', 'FREEZE_VARS').split(',')
]
# Throw exception if optimizer is not defined
if self._CLS_OPTIMIZER not in self._OPTIMIZER_MAP.keys():
raise ValueError("CLASSIFIER_TRAINING OPTIMIZER not defined.")
if self._TPL_OPTIMIZER not in self._OPTIMIZER_MAP.keys():
raise ValueError("TRIPLET_TRAINING OPTIMIZER not defined.")
class ModelTrainer:
def __init__(self, config_path='configs/sample_config.ini'):
"""Read and set configuration from config file (.ini file) and create
keras.Model object or input function according to configuration. To add
new model, simply add new base model to self._MODEL_MAP.
Args:
config_path: Str, path to config (.ini) file.
Raises:
ValueError: if values in config file does not have the correct type.
ValueError: if optimizer does not exists in predefined map.
"""
# Pre-defined learning rate schedules
self._LR_SCHEDULE_MAP = {
'ExponentialDecay':
tf.keras.optimizers.schedules.ExponentialDecay,
'PiecewiseConstantDecay':
tf.keras.optimizers.schedules.PiecewiseConstantDecay,
'PolynomialDecay':
tf.keras.optimizers.schedules.PolynomialDecay,
}
# Pre-defined optimizers
self._OPTIMIZER_MAP = {
'Adam':
tf.keras.optimizers.Adam,
'RMSprop':
tf.keras.optimizers.RMSprop,
}
# Pre-defined losses
# IMPORTANT: DON'T USE TRIPLET HARD LOSS! EXTREMELY HARD TO TRAIN!
self._LOSS_MAP = {
'CrossEntropy':
tf.keras.losses.CategoricalCrossentropy,
'TripletHard':
tfa.losses.TripletHardLoss,
'TripletSemiHard':
tfa.losses.TripletSemiHardLoss,
}
# Pre-defined metrics
self._METRIC_MAP = {
'Accuracy':
tf.keras.metrics.CategoricalAccuracy,
}
# Get custom dataset
self.datset_builder = DatasetBuilder(config_path=config_path)
self.model_builder = ModelBuilder(config_path=config_path)
# Parse config file
config = configparser.ConfigParser()
config.read(config_path)
# Get classifier training config
self._CLS_CKPT_DIR = config.get('CLASSIFIER_TRAINING', 'CKPT_DIR')
self._CLS_MAX_STEP = config.getint('CLASSIFIER_TRAINING', 'MAX_STEP')
self._CLS_OPTIMIZER = config.get('CLASSIFIER_TRAINING', 'OPTIMIZER')
self._CLS_LR_BOUNDARIES = [
int(boundary.strip()) for boundary in
config.get('CLASSIFIER_TRAINING', 'LR_BOUNDARIES').split(',')
]
self._CLS_LR_VALUES = [
float(value.strip()) for value in
config.get('CLASSIFIER_TRAINING', 'LR_VALUES').split(',')
]
# Get triplet training config
self._TPL_INIT_DIR = config.get('TRIPLET_TRAINING', 'INIT_DIR')
self._TPL_CKPT_DIR = config.get('TRIPLET_TRAINING', 'CKPT_DIR')
self._TPL_CYCLES = config.getint('TRIPLET_TRAINING', 'CYCLES')
self._TPL_EPOCHS = config.getint('TRIPLET_TRAINING', 'EPOCHS')
self._TPL_FILTER_SIZE = config.getint('TRIPLET_TRAINING', 'FILTER_SIZE')
self._TPL_MARGIN = config.getfloat('TRIPLET_TRAINING', 'MARGIN')
self._TPL_OPTIMIZER = config.get('TRIPLET_TRAINING', 'OPTIMIZER')
self._TPL_LR_VALUE = config.getfloat('TRIPLET_TRAINING',
'LEARNING_RATE')
self._TPL_FREEZE_VARS = [
var.strip() for var in
config.get('TRIPLET_TRAINING', 'FREEZE_VARS').split(',')
]
# Throw exception if optimizer is not defined
if self._CLS_OPTIMIZER not in self._OPTIMIZER_MAP.keys():
raise ValueError("CLASSIFIER_TRAINING OPTIMIZER not defined.")
if self._TPL_OPTIMIZER not in self._OPTIMIZER_MAP.keys():
raise ValueError("TRIPLET_TRAINING OPTIMIZER not defined.")
def train_classifier(self):
'''Train classifer according to specs in config file.'''
# When training classifier, we uses one-hot encoding as label
self.datset_builder.ONE_HOT = True
# Create full model using model_builder
model, input_name = self.model_builder.create_full_model()
# Sanity check
model.summary()
# Set learning rate schedule
boundaries = self._CLS_LR_BOUNDARIES
values = self._CLS_LR_VALUES
lr_schedule = self._LR_SCHEDULE_MAP['PiecewiseConstantDecay'](
boundaries=boundaries, values=values)
# Use learning reate schedule to create optimizer
optimizer = self._OPTIMIZER_MAP[self._CLS_OPTIMIZER](
learning_rate=lr_schedule)
# Create loss function
loss = self._LOSS_MAP['CrossEntropy'](from_logits=True)
# Add accuracy metrics
accuracy = self._METRIC_MAP['Accuracy']()
model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy])
# Build tf.estimator
estimator = tf.keras.estimator \
.model_to_estimator(keras_model=model, model_dir=self._CLS_CKPT_DIR)
train_spec = tf.estimator.TrainSpec(
input_fn=self.datset_builder.get_train_input_fn(input_name),
max_steps=self._CLS_MAX_STEP)
eval_spec = tf.estimator.EvalSpec(
input_fn=self.datset_builder.get_eval_input_fn(input_name))
# Start training
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def _freeze_vars(self, model):
"""Freeze variables in the model based on regular expressions in
self._TPL_FREEZE_VARS.
Args:
model: tf.keras.Model, the model within which variables are frozen.
"""
# Get regular expressions in config file.
freeze_var_res = self._TPL_FREEZE_VARS
# Get layers that matches regular expression.
freeze_layers = [layer for layer in model.layers if
any(re.match(str(pattern), layer.name) for pattern in
freeze_var_res)]
# Freeze layers.
print('Freezing {} layers.'.format(str(len(freeze_layers))))
for layer in freeze_layers:
print('Freezing layer {}.'.format(layer.name))
layer.trainable = False
def train_triplet_transfer(self):
"""Train encoder with triplet loss according to specs in config file."""
# When training using triplet loss, we avoid using one-hot encoding
self.datset_builder.ONE_HOT = False
# Create full model using model_builder
model, input_name = self.model_builder.create_full_model()
# Sanity check
model.summary()
# Build optimizer
optimizer = self._OPTIMIZER_MAP[self._TPL_OPTIMIZER](self._TPL_LR_VALUE)
# Load initial weights from self._TPL_INIT_DIR
init_dir = self._TPL_INIT_DIR
latest = tf.train.latest_checkpoint(init_dir)
model.load_weights(latest)
# Get ResNet50 model
resnet_model = model.layers[0]
# Freeze specified variables
self._freeze_vars(resnet_model)
# Create loss function
loss = self._LOSS_MAP['TripletSemiHard'](self._TPL_MARGIN)
model.compile(optimizer=optimizer, loss=loss)
# Train triplet model
# In each cycle, a new training dataset with N labels are generated and
# training is carried out for M epochs.
# Total number of cycles = self._TPL_CYCLES
# N = self._TPL_FILTER_SIZE
# M = self._TPL_EPOCHS
for i in range(self._TPL_CYCLES):
print('Cycle #{}'.format(i+1))
train_dataset = self.datset_builder.get_train_dataset(
filter_size=self._TPL_FILTER_SIZE)
history = model.fit(
train_dataset,
epochs=self._TPL_EPOCHS
)
# Store weights every 50 cycles
if (i+1) % 50 == 0:
model.save_weights(self._TPL_CKPT_DIR + '_#{}'.format(i+1))
model.save_weights(self._TPL_CKPT_DIR)
# 2. (alernative to the previous line) Load URLs from a file instead of the web:
#crawler.load_urls(download_folder + "/links.txt")
#crawler.load_urls_from_json(download_folder + "/links.json")
# 3. Save URLs to download them later (optional):
crawler.save_urls(download_folder + "/links.txt")
#crawler.save_urls_to_json(download_folder + "/links.json")
# 4. Download the images:
crawler.download_images(target_folder=download_folder)
### Build the dataset ###
from dataset_builder import DatasetBuilder
dataset_builder = DatasetBuilder()
# 1. rename the downloaded images:
source_folder = download_folder
target_folder = download_folder + "_renamed"
dataset_builder.rename_files(source_folder, target_folder)
#dataset_builder.rename_files(source_folder, target_folder, extensions=('.jpg', '.jpeg', '.png', '.gif'))
# 2. Resize the images:
source_folder = download_folder
target_folder = download_folder + "_resized"
dataset_builder.reshape_images(source_folder, target_folder)
#dataset_builder.reshape_images(source_folder, target_folder, width=64, height=64)
# 3. Crop the images:
source_folder = download_folder