def autoconfig(config):
print("setting autoconfig")
temp_loader = DataManager(os.path.join(".", "utils"))
auto_configuration = temp_loader.load_json("autoconfig")
if config['data_set'] in auto_configuration["dataset"]:
for key, value in auto_configuration["dataset"][
config['data_set']].items():
setattr(config, key, value)
if config['model'] in auto_configuration["model"]:
for key, value in auto_configuration["model"][config['model']].items():
setattr(config, key, value)
if config['l0']:
for key, value in auto_configuration["l0"].items():
# setattr(config, key, value)
config['bla'] = value
print(config[key])
if config['prune_criterion'] in auto_configuration:
for key, value in auto_configuration[
config['prune_criterion']].items():
setattr(config, key, value)
if config['hoyer_square']:
for key, value in auto_configuration["hoyer_square"].items():
setattr(config, key, value)
if config['group_hoyer_square']:
for key, value in auto_configuration["group_hoyer_square"].items():
setattr(config, key, value)
return config
def evaluate(test_data, net, filepath=None):
net.eval()
test_data_manager = DataManager(len(test_data),
num_epoch=1,
batch_size=batch_size)
num_batch = test_data_manager.num_batch_per_epoch
pred = []
truth = []
logits = []
for batch in range(num_batch):
t0 = time.time()
batch_data = test_data_manager.get_batch(test_data)
accel = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[0] for item in batch_data]),
[0, 2, 1])).cuda())
alphanum = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[1] for item in batch_data]),
[0, 2, 1])).cuda())
special = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[2] for item in batch_data]),
[0, 2, 1])).cuda())
labels = [item[3] for item in batch_data]
label_HDRS = [item[0] for item in labels]
label_YMRS = [item[1] for item in labels]
timestamp = Variable(
torch.from_numpy(np.asarray([item[4] for item in batch_data
])).cuda().float())
users = Variable(
torch.from_numpy(np.asarray([item[5]
for item in batch_data])).cuda())
t1 = time.time()
outputs = net.forward(accel, alphanum, special, timestamp, users)
if mode == 'clf':
probs = torch.sigmoid(outputs)
logits += list(probs.data.cpu().numpy())
pred += list(probs.data.cpu().numpy() > 0.5)
truth += label_HDRS
else:
pred += list(outputs.data.cpu().numpy())
if mode == 'rgs_hdrs':
truth += label_HDRS
else:
truth += label_YMRS
if mode == 'clf':
acc = accuracy_score(y_true=truth, y_pred=pred)
f1 = f1_score(y_true=truth, y_pred=pred)
if filepath is not None:
save_log_data([logits, truth], filepath)
return [acc, f1]
else:
rmse = np.sqrt(mean_squared_error(y_true=truth, y_pred=pred))
return [rmse]
def setup(self):
log.info("PRECONDITIONS: Prepare test data")
header = DataManager.get_header_with_profile_data(base_fb.page_header)
body = DataManager.get_body_with_message(config.message)
log.info("PRECONDITIONS: Publish new message")
response = Request.post(config.base_url, header, body, base_fb.page_id,
routes.feed)
self.msg_id = response['id']
log.info("PRECONDITIONS: Done")
def _train(args):
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s [%(levelname)s] => %(message)s',
handlers=[
logging.FileHandler(filename=args['prefix'] +
'_{}_{}_{}_{}.log'.format(
args['model_name'], args['convnet_type'],
args['init_cls'], args['increment'])),
logging.StreamHandler(sys.stdout)
])
logging.info('Seed: {}'.format(args['seed']))
logging.info('Model: {}'.format(args['model_name']))
logging.info('Convnet: {}'.format(args['convnet_type']))
_set_device(args)
data_manager = DataManager(args['dataset'], args['shuffle'], args['seed'],
args['init_cls'], args['increment'])
model = factory.get_model(args['model_name'], args)
curve = []
for task in range(data_manager.nb_tasks):
logging.info('All params: {}'.format(count_parameters(model._network)))
logging.info('Trainable params: {}'.format(
count_parameters(model._network, True)))
model.incremental_train(data_manager)
accy = model.eval_task()
model.after_task()
logging.info(accy)
curve.append(accy['total'])
logging.info('Curve: {}\n'.format(curve))
def TestSamples(samples,
gts,
net,
tester,
device='cuda',
I_only=True,
batch_size=K.BatchSize,
SNR_generate=None):
sum_loss = 0
i1 = 0
while i1 < len(samples):
if i1 + batch_size < len(samples):
i2 = i1 + batch_size
else:
i2 = len(samples)
batch_X = samples[i1:i2]
if SNR_generate:
batch_X = DataManager.add_complex_gaussian_noise(
batch_X, SNR=SNR_generate(), I_only=I_only)
batch_X = batch_X.reshape(i2 - i1, 1 if I_only else 2, -1)
batch_X = torch.tensor(batch_X, dtype=torch.float32, device=device)
cpu_batch_Y = gts[i1:i2]
batch_Y = torch.tensor(cpu_batch_Y, dtype=torch.float32, device=device)
# * Forward
loss, PR = net.get_cross_entropy_loss(batch_X,
batch_Y,
need_PR=True,
is_expanded_target=True)
sum_loss += loss * (i2 - i1)
tester.update_confusion_matrix(PR.cpu().numpy(), cpu_batch_Y)
i1 += batch_size
tester.measure()
return float(sum_loss) / len(samples), tester.micro_avg_precision
def main():
y_trues = []
y_preds = []
for i in xrange(0, len(model_types)):
model_type = model_types[i]
# test dataset
model_test_dataset = 'dataset_' + model_type
# path to saved model files
saved_model_weights_path = './trained_for_pred/' + \
model_type + '/model/Best-weights.h5'
saved_model_arch_path = './trained_for_pred/' + \
model_type + '/model/scratch_model.json'
test_data_dir = './datasets/' + model_test_dataset + '/test'
# init DataManager class
dataManager = DataManager(img_height, img_width)
# load model
print("===================== load model =========================")
model = load_model(saved_model_arch_path, saved_model_weights_path)
# get test data
print("===================== load data =========================")
test_data = dataManager.get_test_data(test_data_dir)
# start the eval process
print("===================== start eval =========================")
y_true = test_data.classes
# Confution Matrix and Classification Report
Y_pred = model.predict_generator(test_data,
num_of_test_samples // batch_size)
y_pred = np.argmax(Y_pred, axis=1)
y_trues.append(y_true)
y_preds.append(y_pred)
# init PlotData class
plotData = PlotData()
# Compute ROC curve and ROC area for each class
plotData.plot_roc(y_trues, y_preds, colors, linestyles, legends,
save_plt_roc)
def autoconfig(config):
print("setting autoconfig")
temp_loader = DataManager(os.path.join(".", "utils"))
auto_configuration = temp_loader.load_json("autoconfig")
if config.data_set in auto_configuration["dataset"]:
for key, value in auto_configuration["dataset"][config.data_set].items():
setattr(config, key, value)
if config.model in auto_configuration["model"]:
for key, value in auto_configuration["model"][config.model].items():
setattr(config, key, value)
if config.l0:
for key, value in auto_configuration["l0"].items():
setattr(config, key, value)
if config.prune_criterion in auto_configuration:
for key, value in auto_configuration[config.prune_criterion].items():
setattr(config, key, value)
if config.hoyer_square:
for key, value in auto_configuration["hoyer_square"].items():
setattr(config, key, value)
if config.group_hoyer_square:
for key, value in auto_configuration["group_hoyer_square"].items():
setattr(config, key, value)
def _train(args):
logfilename = '{}_{}_{}_{}_{}_{}_{}'.format(
args['prefix'], args['seed'], args['model_name'], args['convnet_type'],
args['dataset'], args['init_cls'], args['increment'])
logging.basicConfig(level=logging.INFO,
format='%(asctime)s [%(filename)s] => %(message)s',
handlers=[
logging.FileHandler(filename=logfilename + '.log'),
logging.StreamHandler(sys.stdout)
])
_set_random()
_set_device(args)
print_args(args)
data_manager = DataManager(args['dataset'], args['shuffle'], args['seed'],
args['init_cls'], args['increment'])
model = factory.get_model(args['model_name'], args)
cnn_curve, nme_curve = {'top1': [], 'top5': []}, {'top1': [], 'top5': []}
for task in range(data_manager.nb_tasks):
logging.info('All params: {}'.format(count_parameters(model._network)))
logging.info('Trainable params: {}'.format(
count_parameters(model._network, True)))
model.incremental_train(data_manager)
cnn_accy, nme_accy = model.eval_task()
model.after_task()
if nme_accy is not None:
logging.info('CNN: {}'.format(cnn_accy['grouped']))
logging.info('NME: {}'.format(nme_accy['grouped']))
cnn_curve['top1'].append(cnn_accy['top1'])
cnn_curve['top5'].append(cnn_accy['top5'])
nme_curve['top1'].append(nme_accy['top1'])
nme_curve['top5'].append(nme_accy['top5'])
logging.info('CNN top1 curve: {}'.format(cnn_curve['top1']))
logging.info('CNN top5 curve: {}'.format(cnn_curve['top5']))
logging.info('NME top1 curve: {}'.format(nme_curve['top1']))
logging.info('NME top5 curve: {}\n'.format(nme_curve['top5']))
else:
logging.info('No NME accuracy.')
logging.info('CNN: {}'.format(cnn_accy['grouped']))
cnn_curve['top1'].append(cnn_accy['top1'])
cnn_curve['top5'].append(cnn_accy['top5'])
logging.info('CNN top1 curve: {}'.format(cnn_curve['top1']))
logging.info('CNN top5 curve: {}\n'.format(cnn_curve['top5']))
def evaluate(dataset, input_shape, model_path):
if os.path.exists(model_path):
model = load_model(model_path)
else:
raise Exception("The model doesn't exist!")
truth, prediction = [], []
print("load data ...")
if dataset == 'RAF':
# loading dataset
data_loader = DataManager(dataset, image_size=input_shape[:2])
faces, emotions, usages = data_loader.get_data()
faces = process_img(faces)
train_data, val_data = split_raf_data(faces, emotions, usages)
data, label = val_data
count = len(label)
correct = 0
for i, d in enumerate(data):
if i % 200 == 0:
plot_progress(i, count)
d = np.expand_dims(d, 0)
emotion_values = model.predict(d)
emotion_label_arg = np.argmax(emotion_values)
p = int(emotion_label_arg)
t = int(np.argmax(label[i]))
if p == t:
correct += 1
prediction.append(p)
truth.append(t)
accuracy = correct / float(count)
print(correct, count, accuracy)
else:
raise Exception("RAF only!")
return truth, prediction, accuracy
def _test_sample_vae():
train_dataset = LyricsRawDataset(os.path.join('local_data', 'data'), TRAIN_SET, genre=Genre.Rock)
vae = SentenceVAE(
vocab_size=train_dataset.vocab_size,
sos_idx=train_dataset.sos_idx,
eos_idx=train_dataset.eos_idx,
pad_idx=train_dataset.pad_idx,
unk_idx=train_dataset.unk_idx,
max_sequence_length=50,
embedding_size=300,
rnn_type='gru',
hidden_dim=64,
latent_size=32).cuda()
# vae: SentenceVAE
datamanager = DataManager("./local_data/results/2019-09-26_19.18.29")
loaded = datamanager.load_python_obj("models/model_best")
state_dict = 0
for state_dict in loaded.values():
state_dict = state_dict
vae.load_state_dict(state_dict)
vae.eval()
y = vae.sample()
for sen in y:
string = ""
for num in sen:
string += (train_dataset.i2w[str(num.item())])
print(string)
def __init__(self, folder, set_name, normalize: bool = False, **kwargs):
super(LyricsDataset, self).__init__()
self.normalize = normalize
data_manager = DataManager(folder)
# load the song entries pickle
self._song_entries = self.setup(data_manager, set_name)
self.set_name = set_name
# assert that the embedding folder exists inside the passed folder
embeddings_folder_path = os.path.join(folder, 'embeddings')
assert os.path.exists(embeddings_folder_path)
# assert that the embedding file for this set exists inside the embedding folder
self._embeddings_file_path = os.path.join(
embeddings_folder_path, f'embeddings.{set_name}.hdf5')
assert os.path.exists(self._embeddings_file_path)
print('-- Loaded dataset:', self.set_name, '- size:', self.__len__())
def main(args):
# Load data
dm = DataManager(data_dir=args.data_dir)
outputs = []
#iterate text and run tone analyzer
for idx, s in enumerate(dm._raw_samples):
tag = 'ordinary' if dm._sentiments[idx, 0] else 'nostalgic'
print('tag:{0}'.format(tag))
print('{0} text:{1}\n'.format(idx, s))
tone_dict = {
'text':
s,
'tag':
tag,
'tone':
tone_analyzer.tone(text=s, tones='emotion',
sentences=False)['document_tone']
}
outputs.append(tone_dict)
with open('output/tone_analyzer.json', 'w') as fout:
json.dump(outputs, fout)
def __init__(self,
folder,
set_name,
create_data=False,
genre: Genre = None,
**kwargs):
super().__init__()
self.data_dir = folder
self.split = set_name
self.max_sequence_length = kwargs.get('max_sequence_length', 500)
self.min_occ = kwargs.get('min_occ', 3)
self.genre = genre
data_manager = DataManager(folder)
# load the song entries pickle
song_entries = self.setup(data_manager, set_name)
self.data_file = f'lyrics.{set_name}.{genre}.json'
self.vocab_file = f'lyrics.vocab.json'
if create_data:
print("Creating new %s ptb data." % set_name.upper())
self._create_data(song_entries)
elif not os.path.exists(os.path.join(self.data_dir, self.data_file)):
print("%s preprocessed file not found at %s. Creating new." %
(set_name.upper(), os.path.join(self.data_dir,
self.data_file)))
self._create_data(song_entries)
else:
self._load_data()
print('-- Loaded dataset:', set_name, '- size:', self.__len__())
def main():
# init DataManager class
dataManager = DataManager(img_height, img_width)
# init PlotData class
plotData = PlotData()
# load model
print("===================== load model =========================")
model = load_model()
# get test data
print("===================== load data =========================")
test_data = dataManager.get_test_data(test_data_dir)
# start the eval process
print("===================== start eval =========================")
y_true = test_data.classes
# Confution Matrix and Classification Report
Y_pred = model.predict_generator(test_data, num_of_test_samples // batch_size)
y_pred = np.argmax(Y_pred, axis=1)
# plot confusion matrix
cm = confusion_matrix(y_true, y_pred)
plotData.plot_confusion_matrix(
cm, cm_plot_labels, save_plt_cm, title='Confusion Matrix')
plotData.plot_confusion_matrix(
cm, cm_plot_labels, save_plt_normalized_cm, normalize=True, title='Normalized Confusion Matrix')
# Compute ROC curve and ROC area for each class
roc_auc = plotData.plot_roc(y_true, y_pred, save_plt_roc)
mae = mean_absolute_error(y_true, y_pred)
mse = mean_squared_error(y_true, y_pred)
accuracy = accuracy_score(y_true, y_pred)
print('mean absolute error: ' + str(mae))
print('mean squared error: ' + str(mse))
print('Area Under the Curve (AUC): ' + str(roc_auc))
c_report = classification_report(
y_true, y_pred, target_names=cm_plot_labels)
print(c_report)
delete_file(save_eval_report)
with open(save_eval_report, 'a') as f:
f.write('\n\n')
f.write('******************************************************\n')
f.write('************** Evalaluation Report ***************\n')
f.write('******************************************************\n')
f.write('\n\n')
f.write('- Accuracy Score: ' + str(accuracy))
f.write('\n\n')
f.write('- Mean Absolute Error (MAE): ' + str(mae))
f.write('\n\n')
f.write('- Mean Squared Error (MSE): ' + str(mse))
f.write('\n\n')
f.write('- Area Under the Curve (AUC): ' + str(roc_auc))
f.write('\n\n')
f.write('- Confusion Matrix:\n')
f.write(str(cm))
f.write('\n\n')
f.write('- Normalized Confusion Matrix:\n')
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
f.write(str(cm))
f.write('\n\n')
f.write('- Classification report:\n')
f.write(str(c_report))
f.close()
train_validation = ['train', 'validation']
data = pd.read_csv(train_log_data_path)
acc = data['acc'].values
val_acc = data['val_acc'].values
loss = data['loss'].values
val_loss = data['val_loss'].values
# plot metrics to the stats dir
plotData.plot_2d(acc, val_acc, 'epoch', 'accuracy',
'Model Accuracy', train_validation, save_plt_accuracy)
plotData.plot_2d(loss, val_loss, 'epoch', 'loss',
'Model Loss', train_validation, save_plt_loss)
plotData.plot_model_bis(data, save_plt_learning)
'''
def main():
# Init the class DataManager
print("===================== load data =========================")
dataManager = DataManager(img_height, img_width)
# Get data
train_data, validation_data = dataManager.get_train_data(
train_data_dir, validation_data_dir, train_batch_size, val_batch_size)
# Get class name:id
label_map = (train_data.class_indices)
# save model class id
with open(saved_model_classid_path, 'w') as outfile:
json.dump(label_map, outfile)
# Init the class ScratchModel
scratchModel = ScratchModel(image_shape, class_number)
# Get model architecture
print(
"===================== load model architecture ========================="
)
model = scratchModel.get_model_architecture()
# plot the model
plot_model(model, to_file=model_png) # not working with windows
# serialize model to JSON
model_json = model.to_json()
with open(saved_model_arch_path, "w") as json_file:
json_file.write(model_json)
print("===================== compile model =========================")
# Compile the model
model = scratchModel.compile_model(model, model_loss_function,
model_optimizer_rmsprop, model_metrics)
# prepare weights for the model
Kernels = np.empty([5, 5, 4], dtype=np.float32)
for i in xrange(0, 5):
row = np.empty([5, 4], dtype=np.float32)
for j in xrange(0, 5):
row[j][0] = KV[i][j]
row[j][1] = KM[i][j]
row[j][2] = GH[i][j]
row[j][3] = GV[i][j]
Kernels[i] = row
preprocess_weights = np.reshape(Kernels, (5, 5, 1, 4))
model.set_weights([preprocess_weights])
# model.load_weights(best_weights)
# Re-compile model with the setted wegiht
model = scratchModel.compile_model(model, model_loss_function,
model_optimizer_rmsprop, model_metrics)
# Delete the last summary file
delete_file(model_summary_file)
# Add the new model summary
model.summary(print_fn=save_summary)
# Prepare callbacks
csv_log = callbacks.CSVLogger(train_log_path, separator=',', append=False)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
checkpoint = callbacks.ModelCheckpoint(train_checkpoint_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
tensorboard = TensorBoard(log_dir=model_tensorboard_log +
"{}".format(time()))
callbacks_list = [csv_log, tensorboard, checkpoint]
print(
"===================== start training model =========================")
# start training
history = model.fit_generator(
train_data,
steps_per_epoch=num_of_train_samples // train_batch_size,
epochs=num_of_epoch,
validation_data=validation_data,
validation_steps=num_of_validation_samples // val_batch_size,
verbose=1,
callbacks=callbacks_list)
print(history)
print(
"========================= training process completed! ==========================="
)
FLAGS.summaries_dir,
datetime.datetime.now().strftime('%d_%b_%Y-%H_%M_%S'))
train_writer = tf.summary.FileWriter(summaries_dir + '/train')
validation_writer = tf.summary.FileWriter(summaries_dir + '/validation')
# Save configuration
FLAGS._parse_flags()
config = FLAGS.__dict__['__flags']
with open('{}/config.pkl'.format(summaries_dir + '/train'), 'wb') as f:
pickle.dump(config, f)
# Prepare data and build TensorFlow graph
dm = DataManager(data_dir=FLAGS.data_dir,
stopwords_file=FLAGS.stopwords_file,
sequence_len=FLAGS.sequence_len,
test_size=FLAGS.test_size,
val_samples=FLAGS.batch_size,
n_samples=FLAGS.n_samples,
random_state=FLAGS.random_state)
# create tsv file for word embedding visualiation in tensorboard
vocab_sorted = sorted([(k, v[0], v[1]) for k, v in dm._vocab.items()],
key=lambda x: x[1])
with open(
os.path.join(summaries_dir + '/train',
"metadata_" + str(FLAGS.n_samples) + ".tsv"), 'wb') as f:
f.write('Word\tFrequency\n' +
'\n'.join([(w[0] if len(w[0]) else 'NONE') + '\t' + str(w[2])
for w in vocab_sorted]))
nn = NeuralNetwork(hidden_size=[FLAGS.hidden_size],
class IncrementalLearning:
def __init__(self):
self.args = ArgumentManager().get_args(parser_type='incremental')
self.dsae = True
# === Data
self.dataset = self.args.dataset
if self.dataset == 'miniImageNet' or 'cnbc-face' or 'cub-200':
self.decoder_output_size = 6 if self.dsae else 84
self.decoder_scale_factor = 3
elif self.dataset == 'cifar100':
self.decoder_output_size = 4 if self.dsae else 32
self.decoder_scale_factor = 2
elif self.dataset == 'casia-face':
self.decoder_output_size = 8 if self.dsae else 128
self.decoder_scale_factor = 3
else:
raise NotImplementedError
torch.manual_seed(self.args.seed)
print('==============\nArgs:{}\n=============='.format(self.args))
if USE_GPU:
print('Currently using GPU: {}-{}'.format(
torch.cuda.current_device(), torch.cuda.get_device_name()))
cudnn.benchmark = True
torch.cuda.manual_seed_all(self.args.seed)
else:
print('Currently using CPU (GPU is highly recommended)')
# === Encoder & Decoder
self.encoder = create_encoder(self.args, use_avgpool=True, is_snail=False)
print('Encoder:', self.encoder)
if self.dsae:
self.decoder = create_decoder(self.args, out_dim=256, fm_level=3)
else:
self.decoder = create_decoder(self.args, out_dim=3, fm_level=-1)
print('Decoder:', self.decoder)
if self.args.load: # Loading pre-trained checkpoint
ckp_path = osp.join(self.args.load_dir, self.dataset, 'pretrain', self.args.encoder, 'best_model.ckp')
print('Loading checkpoint from {}'.format(ckp_path))
ckp = torch.load(ckp_path)
encoder_state_dict = ckp['encoder_state_dict']
decoder_state_dict = ckp['decoder_state_dict']
self.encoder.load_state_dict(encoder_state_dict, strict=False)
self.decoder.load_state_dict(decoder_state_dict, strict=True)
# === MemoryK
self.m_sz = self.args.memory_size # 1024
self.m_key_dim = self.args.memory_key_dim # 128
self.m_K = self.args.memory_K
self.need_norm = self.args.need_norm
self.memory = Memory(mem_size=self.m_sz, key_dim=self.m_key_dim, tau=0.95, need_norm=self.need_norm)
self.trainable_params = chain(
self.encoder.parameters(), self.memory.parameters(), self.decoder.parameters())
self.optimizer_all = create_optim(
optim_name='sgd', lr=1e-3, params=self.trainable_params)
self.mse_loss = nn.MSELoss(reduction='mean')
# self.scheduler_encoder = MultiStepLR(
# self.optimizer_encoder, milestones=self.args.lr_decay_steps, gamma=0.1)
self.gamma = 0.2
self.param_frozen = False # False default
self.cur_session = None
if USE_GPU:
self.encoder = self.encoder.cuda()
self.memory = self.memory.cuda()
self.decoder = self.decoder.cuda()
self.eval_dataloaders = []
current_time = datetime.datetime.now().strftime('%b%d_%H-%M-%S')
self.writer = SummaryWriter(log_dir=osp.join('runs', 'incremental', current_time))
self.data_manager = DataManager(self.args, use_gpu=USE_GPU)
def get_dataloader(self, session):
if session == 0:
train_loader, eval_loader = self.data_manager.get_dataloaders()
else: # Incremental sessions
if self.dataset == 'miniImageNet':
train_loader = self.data_manager.get_dataloaders(session=session, is_fewshot=True)
eval_loader = self.data_manager.get_dataloaders(session=session, is_fewshot=False)
elif self.dataset in ('cifar100', 'cub-200', 'casia-face', 'cnbc-face'):
train_loader, eval_loader = self.data_manager.get_dataloaders(
session=session, is_fewshot=True)
else:
raise NotImplementedError
return train_loader, eval_loader
def run(self, start_session=0, end_session=8):
if start_session > 0: # Load
load_dir = osp.join(self.args.save_dir, self.dataset, 'incr', self.args.encoder,
'session' + str(start_session - 1), 'best_model.ckp')
print('Start session > 0, loading checkpoint from:', load_dir)
ckp = torch.load(load_dir)
self.encoder.load_state_dict(ckp['encoder_state_dict'])
self.memory.load_state_dict(ckp['memory_state_dict'])
self.decoder.load_state_dict(ckp['decoder_state_dict'])
# Evaluate seen classes
for passed_session in range(start_session):
# if passed_session == 0:
# _, eval_loader = self.data_manager.get_dataloaders()
# else:
# eval_loader = self.data_manager.get_dataloaders(session=passed_session)
_, eval_loader = self.get_dataloader(passed_session)
self.eval_dataloaders.append(eval_loader)
# self._eval_session(start_session - 1)
for sess in range(start_session, end_session + 1):
if sess > 0 and not self.param_frozen:
for param in self.encoder.parameters():
param.requires_grad = False
for param in self.encoder.layer4.parameters():
param.requires_grad = True
self.trainable_params = chain(
self.encoder.parameters(), self.memory.parameters(), self.decoder.parameters())
self.optimizer_all = create_optim(
optim_name='sgd', lr=1e-3,
params=filter(lambda p: p.requires_grad, self.trainable_params))
self.param_frozen = True
print('Encoder frozen.')
self._train_session(sess, use_centroid=True)
self._eval_session(sess, use_centroid=True)
def _train_session(self, session, use_centroid=False):
# assert session in range(0, 9)
print('Training session {}'.format(session))
self.cur_session = session
if session > 0:
self.memory.del_noisy_slots()
memory_vals = self.memory.m_vals.cpu().numpy()
memory_vals_counter = Counter(memory_vals)
print('memory val:', len(memory_vals_counter), memory_vals_counter.most_common())
# === Data
print('Preparing data {} with session {}...'.format(self.dataset, session))
train_loader, eval_loader = self.get_dataloader(session)
if session == 0:
# train_loader, eval_loader = self.data_manager.get_dataloaders()
max_epoch = self.args.max_epoch_sess0
m_replays = None
else:
# if self.dataset == 'miniImageNet':
# train_loader = self.data_manager.get_dataloaders(session=session, is_fewshot=True)
# eval_loader = self.data_manager.get_dataloaders(session=session, is_fewshot=False)
# elif self.dataset == 'cifar100' or 'cub-200' or 'casia-face':
# train_loader, eval_loader = self.data_manager.get_dataloaders(
# session=session, is_fewshot=True)
# else:
# raise NotImplementedError
# Memory replay data
m_keys, m_targets = self._get_nonempty_memory_slots()
# m_keys = m_keys.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 3, 3) # [nonempty, key_dim, 3, 3]
m_replays = (m_keys, m_targets)
max_epoch = 20 # Todo
print('Num of batches of train loader:', len(train_loader))
self.eval_dataloaders.append(eval_loader)
start_time = time.time()
train_time = 0
best_epoch = 0
# best_acc = self._eval_epoch(epoch=0, eval_loader=eval_loader)
best_acc = -1
best_state = None
# === Train
for epoch in range(1, max_epoch + 1):
if session == 0:
if epoch == 1:
adjust_lr(self.optimizer_all, 1e-3)
if epoch > 10:
adjust_lr(self.optimizer_all, 1e-4)
else:
if epoch == 1:
adjust_lr(self.optimizer_all, 1e-4)
if epoch > 15:
adjust_lr(self.optimizer_all, 1e-5)
cur_lr = self.optimizer_all.param_groups[0]['lr']
self.writer.add_scalar('Learning rate', cur_lr, global_step=epoch)
epoch_time_start = time.time()
self._train_epoch(epoch, cur_lr, train_loader, m_replays)
train_time += round(time.time() - epoch_time_start)
# === Eval on current session's dataloader only
if epoch == self.args.max_epoch_sess0 or epoch % self.args.eval_per_epoch == 0:
if use_centroid:
self.memory.upd_centroids()
acc = self._eval_epoch(epoch, eval_loader, use_centroid=use_centroid)
# === Save checkpoint
is_best = acc > best_acc
if is_best or epoch == 2 or epoch % self.args.save_per_epoch == 0:
state = {
'encoder_state_dict': self.encoder.state_dict(),
'memory_state_dict': self.memory.state_dict(),
'decoder_state_dict': self.decoder.state_dict(),
'acc': acc,
'session': session,
'epoch': epoch,
}
file_path = osp.join(
self.args.save_dir, self.dataset, 'incr', self.args.encoder,
'session' + str(session), 'ckp_ep' + str(epoch) + '.ckp')
if epoch == 2:
pass
# save_checkpoint(state, False, file_path)
else:
save_checkpoint(state, is_best, file_path)
if is_best:
best_acc = acc
best_epoch = epoch
best_state = copy.deepcopy(state)
print('==> Test best accuracy {:.2%}, achieved at epoch {}'.format(
best_acc, best_epoch))
torch.cuda.empty_cache()
# Load best checkpoint
self.encoder.load_state_dict(best_state['encoder_state_dict'])
self.memory.load_state_dict(best_state['memory_state_dict'])
self.decoder.load_state_dict(best_state['decoder_state_dict'])
elapsed = str(datetime.timedelta(seconds=round(time.time() - start_time)))
train_time = str(datetime.timedelta(seconds=train_time))
print('Session {} finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}'.format(
session, elapsed, train_time))
# print("==========\nArgs:{}\n==========".format(self.args))
memory_vals = self.memory.m_vals.cpu().numpy()
memory_vals_counter = Counter(memory_vals)
print('memory val:', len(memory_vals_counter), memory_vals_counter.most_common())
def _train_epoch(self, epoch, lr, train_loader, memory_replay=None, use_reparam=False):
self.encoder.train()
self.memory.train()
self.decoder.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses_clsf = AverageMeter()
losses_recons = AverageMeter()
accs = AverageMeter()
end = time.time()
for batch_idx, (inputs, targets) in enumerate(train_loader):
# print('targets:', len(targets), targets[targets > 54])
data_time.update(time.time() - end)
if USE_GPU:
inputs = inputs.cuda()
targets = targets.cuda()
bs = inputs.shape[0]
if len(inputs.shape) == 5: # episode, batch size = 1
inputs = inputs.squeeze(0) # [25, 3, 84, 84]
targets = targets.squeeze(0)
bs = inputs.shape[0]
# === Encoder & Decoder forward
outputs, fms = self.encoder(inputs, return_fm=True)
img_recons = self.decoder(fms[3], scale_factor=self.decoder_scale_factor,
out_size=self.decoder_output_size)
if self.dsae:
loss_recons = self.mse_loss(img_recons, fms[2]) # reconstruction loss
else:
loss_recons = self.mse_loss(img_recons, inputs)
# === MemoryK forward
# loss_e = torch.tensor(0.)
preds, loss_memory = self.memory(outputs, targets)
acc = preds.eq(targets).sum().float() / bs
accs.update(acc.item(), bs)
loss_all = self.gamma * loss_memory + (1 - self.gamma) * loss_recons
self.optimizer_all.zero_grad()
loss_all.backward()
self.optimizer_all.step()
if batch_idx % 90 == 0:
print(batch_idx, '; memory loss:', loss_memory.item(),
'; decoder loss:', loss_recons.item())
losses_clsf.update(loss_memory.item(), bs)
losses_recons.update(loss_recons.item(), bs)
batch_time.update(time.time() - end)
end = time.time()
# memory_replay = None
if memory_replay is not None:
with torch.no_grad():
m_inputs, m_targets = memory_replay # [nonempty, key_dim]
# m_inputs = m_inputs.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 3, 3) # [nonempty, key_dim, 3, 3]
m_inputs_aug = []
m_targets_aug = []
n_classes = 60 + self.args.n_novel * (self.cur_session - 1)
rand_C_classes = np.random.choice(n_classes, self.args.n_novel, replace=False)
for v in rand_C_classes: # rand_C_classes: # range(n_classes) for all classes
m_inputs_v = m_inputs[torch.eq(m_targets, v)]
# print('m_inputs_v:', m_inputs_v.shape)
if use_reparam: # re-parameterize
m_mean_v = m_inputs_v.mean(dim=0)
m_std_v = m_inputs_v.std(dim=0)
for i in range(self.args.n_shot * 2):
v_aug = torch.normal(mean=m_mean_v, std=m_std_v)
# print('v_aug:', v_aug.shape)
m_inputs_aug.append(v_aug)
m_targets_aug.append(v)
else: # random sample
n_v = m_inputs_v.size(0)
if n_v == 0:
continue
for i in range(self.args.n_shot):
rand_idxs = np.random.choice(n_v, 3, replace=True)
rand_w = F.normalize(torch.rand([3]), p=1, dim=0)
v_aug = (rand_w[0] * m_inputs_v[rand_idxs[0]] +
rand_w[1] * m_inputs_v[rand_idxs[1]] + rand_w[2] * m_inputs_v[rand_idxs[2]])
m_inputs_aug.append(v_aug)
m_targets_aug.append(v)
m_inputs = torch.stack(m_inputs_aug, dim=0)
m_inputs = m_inputs.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 3, 3)
if self.need_norm:
m_inputs = F.normalize(m_inputs, p=2, dim=1)
m_targets = torch.tensor(m_targets_aug, dtype=torch.long)
# Shuffle
sfl_idxs = torch.randperm(m_inputs.size(0))
m_inputs = m_inputs[sfl_idxs]
m_targets = m_targets[sfl_idxs]
print('Memory replay size:', m_inputs.size(0))
m_inputs = self.decoder(m_inputs, scale_factor=self.decoder_scale_factor,
out_size=self.decoder_output_size)
batch_size = 128
n_sample = m_targets.size(0)
n_batch = math.ceil(n_sample / batch_size)
inputs = m_inputs.chunk(chunks=n_batch, dim=0)
targets = m_targets.chunk(chunks=n_batch, dim=0)
print('After chunk, inputs:', inputs[0].shape, '; targets:', targets[0].shape)
m_train_loader = list(zip(inputs, targets))
for batch_idx, (inputs, targets) in enumerate(m_train_loader):
data_time.update(time.time() - end)
if USE_GPU:
inputs = inputs.cuda()
targets = targets.cuda()
# === Encoder & Decoder forward
outputs = self.encoder(inputs, return_fm=False, feed_fm=self.dsae)
img_recons = self.decoder(outputs, scale_factor=self.decoder_scale_factor,
out_size=self.decoder_output_size)
loss_recons = self.mse_loss(img_recons, inputs)
# === MemoryK forward
preds, loss_memory = self.memory(outputs, targets, upd_memory=False)
loss_all = self.gamma * loss_memory + (1 - self.gamma) * loss_recons
self.optimizer_all.zero_grad()
loss_all.backward()
self.optimizer_all.step()
acc_avg = accs.avg
loss_c_avg = losses_clsf.avg
loss_r_avg = losses_recons.avg
self.writer.add_scalar('Loss/train/Classification', loss_c_avg, global_step=epoch)
self.writer.add_scalar('Loss/train/Reconstruction', loss_r_avg, global_step=epoch)
print(
'-Train- Epoch: {}, Lr: {:.6f}, Time: {:.1f}s, Data: {:.1f}s, '
'Loss(C|R): {:.4f} | {:.4f}, Acc: {:.2%}'.format(
epoch, lr, batch_time.sum, data_time.sum, loss_c_avg, loss_r_avg, acc_avg))
@torch.no_grad()
def _eval_session(self, session, use_centroid=False):
assert len(self.eval_dataloaders) == session + 1
if use_centroid:
self.memory.upd_centroids()
accuracies = []
for sess in range(session + 1):
eval_loader_sess = self.eval_dataloaders[sess]
acc_sess = self._eval_epoch(epoch=None, eval_loader=eval_loader_sess, use_centroid=use_centroid)
accuracies.append(acc_sess)
acc_sum = AverageMeter()
for sess in range(session + 1):
acc = accuracies[sess]
if sess == 0:
n_cls = 60 # self.args.n_class
else:
n_cls = self.args.n_novel
acc_sum.update(acc, n_cls)
print('Session {} Evaluation. Overall Acc.: {}'.format(session, acc_sum.avg))
@torch.no_grad()
def _eval_epoch(self, epoch, eval_loader, use_centroid=False):
self.encoder.eval()
self.memory.eval()
self.decoder.eval()
accs = AverageMeter()
losses_clsf = AverageMeter()
losses_recons = AverageMeter()
for batch_idx, (inputs, targets) in enumerate(eval_loader):
if USE_GPU:
inputs = inputs.cuda()
targets = targets.cuda()
outputs, fms = self.encoder(inputs, return_fm=True)
# print('outputs:', outputs.shape)
# Decoder
img_recons = self.decoder(fms[3], scale_factor=self.decoder_scale_factor,
out_size=self.decoder_output_size)
if self.dsae:
loss_recons = self.mse_loss(img_recons, fms[2]) # reconstruction loss
else:
loss_recons = self.mse_loss(img_recons, inputs)
preds, loss_memory = self.memory(outputs, targets, use_centroid=use_centroid)
losses_clsf.update(loss_memory.item(), targets.size(0))
losses_recons.update(loss_recons.item(), targets.size(0))
acc = preds.eq(targets).sum().float() / targets.size(0)
accs.update(acc.item(), targets.size(0))
acc_avg = accs.avg
loss_c_avg = losses_clsf.avg
loss_r_avg = losses_recons.avg
if epoch is not None:
self.writer.add_scalar('Loss/eval/Classification', loss_c_avg, global_step=epoch)
self.writer.add_scalar('Loss/eval/Reconstruction', loss_r_avg, global_step=epoch)
self.writer.add_scalar('Accuracy/eval', acc_avg, global_step=epoch)
print('-Eval- Epoch: {}, Loss(C|R): {:.4f} | {:.4f}, Accuracy: {:.2%}'.format(
epoch, loss_c_avg, loss_r_avg, acc_avg))
return acc_avg
@torch.no_grad()
def _get_nonempty_memory_slots(self):
nonempty_idxs = torch.where(self.memory.m_vals != -1)
m_keys = self.memory.m_keys[nonempty_idxs] # [nonempty, key_dim]
# m_keys = m_keys.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 3, 3) # [nonempty, key_dim, 3, 3]
m_vals = self.memory.m_vals[nonempty_idxs] # [nonempty]
return m_keys, m_vals
def main(model='2cnn_rnn_sin_id', pid=0):
if parse_label:
pl = '_norm'
else:
pl = ''
pl += ('_' + str(pid))
checkpoint_path = checkpoint_dir + model + '_' + mode + ctrl + pl + '.ckpt'
if model == 'fillna':
log_file = result_dir + 'log_ef_long_' + mode + ctrl + pl + '.txt'
else:
log_file = result_dir + 'log_ef_short_' + mode + ctrl + pl + '.txt'
net = CNN_RNN_EF2(use_accel=False)
if model == 'fillna':
train_data = train_data_l
test_data = test_data_l
else:
train_data = train_data_s
test_data = test_data_s
net.cuda()
if mode == 'clf':
criterion = nn.BCEWithLogitsLoss()
else:
criterion = nn.MSELoss()
optimizer = optim.RMSprop(net.parameters(),
lr=learning_rate,
weight_decay=0.001)
print 'training...'
best_epoch = 0
best_acc = 0
best_f1 = 0
best_rmse = 1e10
results = []
train_data_manager = DataManager(len(train_data),
num_epoch=num_epochs,
batch_size=batch_size)
for epoch in range(num_epochs):
running_loss = 0.0
num_batch = train_data_manager.num_batch_per_epoch
net.train()
for batch in range(num_batch):
optimizer.zero_grad()
t0 = time.time()
batch_data = train_data_manager.get_batch(train_data)
merged_data = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[0] for item in batch_data]),
[0, 2, 1])).float().cuda())
accel = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[4] for item in batch_data]),
[0, 2, 1])).float().cuda())
timestamp = Variable(
torch.from_numpy(np.asarray([item[2] for item in batch_data
])).cuda().float())
users = Variable(
torch.from_numpy(np.asarray([item[5]
for item in batch_data])).cuda())
labels = [item[1] for item in batch_data]
label_HDRS = Variable(
torch.from_numpy(np.asarray([item[0] for item in labels
])).cuda()).float()
label_YMRS = Variable(
torch.from_numpy(np.asarray([item[1] for item in labels
])).cuda()).float()
t1 = time.time()
outputs = net.forward(merged_data, accel, timestamp=timestamp)
if mode == 'clf' or mode == 'rgs_hdrs':
loss = criterion(outputs, label_HDRS)
else:
loss = criterion(outputs, label_YMRS)
loss.backward()
optimizer.step()
t2 = time.time()
running_loss += loss.data[0]
if batch % 10 == 9: # print every 10 mini-batches
#print('[%d, %5d] loss: %.3f, data time: %.3f, train time: %.3f' %
# (epoch + 1, batch + 1, running_loss/10, t1 - t0, t2 - t1))
running_loss = 0.0
res = evaluate(test_data, net)
results.append(res)
#print 'rmse:', res[0]
if res[0] < best_rmse:
best_rmse = res[0]
best_epoch = epoch + 1
#torch.save(net.state_dict(), checkpoint_path)
#print 'best: epoch %d, rmse: %f' % (best_epoch, best_rmse)
if mode == 'clf':
print 'best: epoch %d, acc: %f, f1: %f' % (best_epoch, best_acc,
best_f1)
else:
print 'best: epoch %d, rmse: %f' % (best_epoch, best_rmse)
save_log_data(results, log_file)
#print 'program finished'
return best_rmse
def _train(args):
logfilename = '{}_{}_{}_{}_{}_{}_{}'.format(
args['prefix'], args['seed'], args['model_name'], args['convnet_type'],
args['dataset'], args['init_cls'], args['increment'])
'''@Author:defeng
{
"prefix": "reproduce",
"dataset": "cifar100",
"memory_size": 2000,
"memory_per_class": 20,
"fixed_memory": true,
"shuffle": true,
"init_cls": 50,
"increment": 10, #increase $increment classes each task. see "# Grouped accuracy" in toolkit.py
"model_name": "UCIR",
"convnet_type": "cosine_resnet32",
"device": ["0"],
"seed": [30]
}
'''
logging.basicConfig(level=logging.INFO,
format='%(asctime)s [%(filename)s] => %(message)s',
handlers=[
logging.FileHandler(filename=logfilename + '.log'),
logging.StreamHandler(sys.stdout)
])
'''@Author:defeng
see for details: https://www.cnblogs.com/xianyulouie/p/11041777.html
26 May 2021 (Wednesday)
format: %(filename)s enables output like this "2021-05-26 22:01:34,371 [*ucir.py*]" and we can know which file \
a certain output come from.
'''
'''@Author:defeng
set random seed and cuda devices
'''
_set_random()
_set_device(args)
print_args(args)
'''@Author:defeng
*set: dataset and model.*
'''
data_manager = DataManager(args['dataset'], args['shuffle'], args['seed'],
args['init_cls'], args['increment'])
model = factory.get_model(args['model_name'], args)
'''@Author:defeng
the actual work for getting model ready is done by the .py files in the "models" folder.
'''
'''@Author:defeng
cnn: softmax prediction
nme: nearest-mean-of-neightbors prediction
see ucir paper "Baselines" for detail.
'''
cnn_curve, nme_curve = {'top1': [], 'top5': []}, {'top1': [], 'top5': []}
for task in range(data_manager.nb_tasks):
logging.info('All params: {}'.format(count_parameters(model._network)))
logging.info('Trainable params: {}'.format(
count_parameters(model._network, True)))
model.incremental_train(data_manager) #train
cnn_accy, nme_accy = model.eval_task() #val
model.after_task() #post-processing
if nme_accy is not None:
logging.info('CNN: {}'.format(cnn_accy['grouped']))
logging.info('NME: {}'.format(nme_accy['grouped']))
cnn_curve['top1'].append(cnn_accy['top1'])
cnn_curve['top5'].append(cnn_accy['top5'])
nme_curve['top1'].append(nme_accy['top1'])
nme_curve['top5'].append(nme_accy['top5'])
logging.info('CNN top1 curve: {}'.format(cnn_curve['top1']))
logging.info('CNN top5 curve: {}'.format(cnn_curve['top5']))
logging.info('NME top1 curve: {}'.format(nme_curve['top1']))
logging.info('NME top5 curve: {}\n'.format(nme_curve['top5']))
else:
logging.info('No NME accuracy.')
logging.info('CNN: {}'.format(cnn_accy['grouped']))
cnn_curve['top1'].append(cnn_accy['top1'])
cnn_curve['top5'].append(cnn_accy['top5'])
logging.info('CNN top1 curve: {}'.format(cnn_curve['top1']))
logging.info('CNN top5 curve: {}\n'.format(cnn_curve['top5']))
def setup(self, data_manager: DataManager, set_name: str):
x = data_manager.load_python_obj(f'song_lyrics.{set_name}')
x: List[Song]
return [song for song in x if self.genre == song.genre]
# ! Automatic Generated Setting
K.LogDir = os.path.join(
'.', 'log',
f'torch.{os.path.split(K.H5DataDir)[1]}.ICRS.{K.LogDirComment}.log')
K.SummaryDir = os.path.join(K.LogDir, 'summary')
K.TrainValLogFile = os.path.join(K.LogDir, 'train_val.log')
K.SnapshotFileStr = os.path.join(K.LogDir, 'snapshot',
'InceptionResNet1D-{}.snapshot')
K.H5ModuleDataDir = os.path.join(K.H5DataDir, 'h5_module_data')
K.H5TrainTestDataDir = os.path.join(K.H5DataDir, 'h5_train_test_split')
if __name__ == '__main__':
# * data, log manager and saver, tester
data_manager = DataManager(K.H5TrainTestDataDir,
K.H5ModuleDataDir,
I_only=K.IOnly,
down_sample=0)
logger = Logger(K.TrainValLogFile).logger
writer = SummaryWriter(K.SummaryDir)
saver = Saver(K.SnapshotFileStr)
tester = MultiClassificationTester(data_manager.classes_list)
# * build OR recover model, optimizer
# writer.add_graph(net, (dummy_input, ), verbose=True)
if not K.IsRecover:
model_init_dict = K.ModelSettings.update({
'num_input_channels':
1 if K.IOnly else 2,
'batch_size':
K.BatchSize
})
net = InceptionResNet1D(data_manager.classes_num, **K.ModelSettings)
def main(model='2cnn_rnn_sin_id', pid=0):
if parse_label:
pl = '_norm'
else:
pl = ''
pl += ('_' + str(pid))
checkpoint_path = checkpoint_dir + model + '_' + mode + ctrl + pl + '.ckpt'
if model == 'DeeperMood_24h':
log_file = './results/log_DM_24h2_' + mode + ctrl + pl + '.txt'
ab_log = './results/hours_emb_2_' + mode + ctrl + pl + '.txt'
net = CNN_RNN(use_time='24h2')
elif model == '2cnn_rnn_sin_id':
log_file = result_dir + 'log_2cnn_rnn_sin_id_' + mode + ctrl + pl + '.txt'
ab_log = result_dir + 'alpha_beta_id_' + mode + ctrl + pl + '.txt'
net = CNN_RNN(use_time='sin_id', num_user=num_users)
elif model == '2cnn_rnn_sin':
log_file = result_dir + 'log_2cnn_rnn_sin_' + mode + ctrl + pl + '.txt'
net = CNN_RNN(use_time='sin', num_user=num_users)
elif model == '2cnn_rnn':
log_file = result_dir + 'log_2cnn_rnn_' + mode + ctrl + pl + '.txt'
net = CNN_RNN(use_time=None)
elif model == 'cnn_rnn':
log_file = result_dir + 'log_cnn_rnn_' + mode + ctrl + pl + '.txt'
net = CNN_RNN_1()
elif model == 'cnn':
log_file = result_dir + 'log_cnn_' + mode + ctrl + pl + '.txt'
net = AllConv()
elif model == 'cnn2':
log_file = result_dir + 'log_cnn2_' + mode + ctrl + pl + '.txt'
net = AllConv(use_special=False)
else: # model == 'rnn':
log_file = result_dir + 'log_rnn_' + mode + ctrl + pl + '.txt'
net = RNN()
train_data = train_data_d
test_data = test_data_d
net.cuda()
if mode == 'clf':
criterion = nn.BCEWithLogitsLoss()
else:
criterion = nn.MSELoss()
optimizer = optim.RMSprop(net.parameters(),
lr=learning_rate,
weight_decay=0.001)
print('training...')
best_epoch = 0
best_acc = 0
best_f1 = 0
best_rmse = 1e10
results = []
train_data_manager = DataManager(len(train_data),
num_epoch=num_epochs,
batch_size=batch_size)
for epoch in range(num_epochs):
num_batch = train_data_manager.num_batch_per_epoch
net.train()
for batch in range(num_batch):
optimizer.zero_grad()
t0 = time.time()
batch_data = train_data_manager.get_batch(train_data)
accel = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[0] for item in batch_data]),
[0, 2, 1])).cuda())
alphanum = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[1] for item in batch_data]),
[0, 2, 1])).cuda())
special = Variable(
torch.from_numpy(
np.transpose(np.asarray([item[2] for item in batch_data]),
[0, 2, 1])).cuda())
timestamp = Variable(
torch.from_numpy(np.asarray([item[4] for item in batch_data
])).cuda().float())
users = Variable(
torch.from_numpy(np.asarray([item[5]
for item in batch_data])).cuda())
labels = [item[3] for item in batch_data]
label_HDRS = Variable(
torch.from_numpy(np.asarray([item[0] for item in labels
])).cuda()).float()
label_YMRS = Variable(
torch.from_numpy(np.asarray([item[1] for item in labels
])).cuda()).float()
t1 = time.time()
outputs = net.forward(accel, alphanum, special, timestamp, users)
if mode == 'clf' or mode == 'rgs_hdrs':
loss = criterion(outputs, label_HDRS)
else:
loss = criterion(outputs, label_YMRS)
loss.backward()
optimizer.step()
t2 = time.time()
res = evaluate(test_data, net)
results.append(res)
if res[0] < best_rmse:
best_rmse = res[0]
best_epoch = epoch + 1
#torch.save(net.state_dict(), checkpoint_path)
#print 'best: epoch %d, rmse: %f' % (best_epoch, best_rmse)
if mode == 'clf':
print('best: epoch %d, acc: %f, f1: %f' %
(best_epoch, best_acc, best_f1))
else:
print('best: epoch %d, rmse: %f' % (best_epoch, best_rmse))
save_log_data(results, log_file)
if model == '2cnn_rnn_sin_id':
idxs = Variable(
torch.from_numpy(np.asarray([i for i in range(num_users)
])).long().cuda())
alpha = net.alpha(idxs).data
beta = net.beta(idxs).data
gamma = net.gamma(idxs).data
delta = net.delta(idxs).data
with open(ab_log, 'w') as fout:
for i in range(num_users):
fout.write(
str(alpha[i][0]) + '\t' + str(beta[i][0]) + '\t' +
str(gamma[i][0]) + '\t' + str(delta[i][0]) + '\n')
return best_rmse
def __init__(self):
self.args = ArgumentManager().get_args(parser_type='incremental')
self.dsae = True
# === Data
self.dataset = self.args.dataset
if self.dataset == 'miniImageNet' or 'cnbc-face' or 'cub-200':
self.decoder_output_size = 6 if self.dsae else 84
self.decoder_scale_factor = 3
elif self.dataset == 'cifar100':
self.decoder_output_size = 4 if self.dsae else 32
self.decoder_scale_factor = 2
elif self.dataset == 'casia-face':
self.decoder_output_size = 8 if self.dsae else 128
self.decoder_scale_factor = 3
else:
raise NotImplementedError
torch.manual_seed(self.args.seed)
print('==============\nArgs:{}\n=============='.format(self.args))
if USE_GPU:
print('Currently using GPU: {}-{}'.format(
torch.cuda.current_device(), torch.cuda.get_device_name()))
cudnn.benchmark = True
torch.cuda.manual_seed_all(self.args.seed)
else:
print('Currently using CPU (GPU is highly recommended)')
# === Encoder & Decoder
self.encoder = create_encoder(self.args, use_avgpool=True, is_snail=False)
print('Encoder:', self.encoder)
if self.dsae:
self.decoder = create_decoder(self.args, out_dim=256, fm_level=3)
else:
self.decoder = create_decoder(self.args, out_dim=3, fm_level=-1)
print('Decoder:', self.decoder)
if self.args.load: # Loading pre-trained checkpoint
ckp_path = osp.join(self.args.load_dir, self.dataset, 'pretrain', self.args.encoder, 'best_model.ckp')
print('Loading checkpoint from {}'.format(ckp_path))
ckp = torch.load(ckp_path)
encoder_state_dict = ckp['encoder_state_dict']
decoder_state_dict = ckp['decoder_state_dict']
self.encoder.load_state_dict(encoder_state_dict, strict=False)
self.decoder.load_state_dict(decoder_state_dict, strict=True)
# === MemoryK
self.m_sz = self.args.memory_size # 1024
self.m_key_dim = self.args.memory_key_dim # 128
self.m_K = self.args.memory_K
self.need_norm = self.args.need_norm
self.memory = Memory(mem_size=self.m_sz, key_dim=self.m_key_dim, tau=0.95, need_norm=self.need_norm)
self.trainable_params = chain(
self.encoder.parameters(), self.memory.parameters(), self.decoder.parameters())
self.optimizer_all = create_optim(
optim_name='sgd', lr=1e-3, params=self.trainable_params)
self.mse_loss = nn.MSELoss(reduction='mean')
# self.scheduler_encoder = MultiStepLR(
# self.optimizer_encoder, milestones=self.args.lr_decay_steps, gamma=0.1)
self.gamma = 0.2
self.param_frozen = False # False default
self.cur_session = None
if USE_GPU:
self.encoder = self.encoder.cuda()
self.memory = self.memory.cuda()
self.decoder = self.decoder.cuda()
self.eval_dataloaders = []
current_time = datetime.datetime.now().strftime('%b%d_%H-%M-%S')
self.writer = SummaryWriter(log_dir=osp.join('runs', 'incremental', current_time))
self.data_manager = DataManager(self.args, use_gpu=USE_GPU)
TestSamples(samples, gts, net, tester,
I_only=K.IOnly, device=K.Device,
SNRs_generator=K.test_SNRs_generator)
tester.show_confusion_matrix()
process_bar.UpdateBar(i + 1)
# ! Show test result
if not os.path.isdir(K.TestResultPath):
os.makedirs(K.TestResultPath)
tester.show_confusion_matrix(img_save_path=os.path.join(K.TestResultPath, "confusion_matrix.png"))
tester.measure()
tester.show_measure_result(rslt_save_path=os.path.join(K.TestResultPath, "test_result.txt"))
if __name__ == '__main__':
# ! Init saver, sess, and data manager
data_manager = DataManager(K.H5TrainTestDataDir, K.H5ModuleDataDir, I_only=K.IOnly, down_sample=0)
data_manager.init_epoch()
tester = MultiClassificationTester(data_manager.classes_list)
saver = Saver(K.SnapshotFileStr)
net, _ = saver.restore(K.LoadModelNum, model_cls=InceptionResNet1D, optimizer_cls=None, device=K.Device)
with torch.no_grad():
net.eval()
if K.IsCompletelyTest:
CompletelyTest(data_manager, net, tester)
if K.IsErrorInspect:
ErrorInspect(data_manager, net, tester)
RESULTS_DIR = "results"
DATA_DIR = "data"
GITIGNORED_DIR = "gitignored"
IMAGENETTE_DIR = os.path.join(".", "gitignored", "data", "imagenette-320")
IMAGEWOOF_DIR = os.path.join(".", "gitignored", "data", "imagewoof-320")
TINY_IMAGNET_DIR = os.path.join(".", "gitignored", "data", "tiny_imagenet")
CODEBASE_DIR = "codebase"
SUMMARY_DIR = "summary"
OUTPUT_DIR = "output"
MODELS_DIR = "models"
PROGRESS_DIR = "progress"
OUTPUT_DIRS = [OUTPUT_DIR, SUMMARY_DIR, CODEBASE_DIR, MODELS_DIR, PROGRESS_DIR]
DATA_MANAGER = DataManager(os.path.join(WORKING_DIR_PATH, GITIGNORED_DIR))
DATASET_PATH = os.path.join(GITIGNORED_DIR, DATA_DIR)
RESULTS_PATH = os.path.join(DATA_MANAGER.directory, RESULTS_DIR)
# printing
PRINTCOLOR_PURPLE = '\033[95m'
PRINTCOLOR_CYAN = '\033[96m'
PRINTCOLOR_DARKCYAN = '\033[36m'
PRINTCOLOR_BLUE = '\033[94m'
PRINTCOLOR_GREEN = '\033[92m'
PRINTCOLOR_YELLOW = '\033[93m'
PRINTCOLOR_RED = '\033[91m'
PRINTCOLOR_BOLD = '\033[1m'
PRINTCOLOR_UNDERLINE = '\033[4m'
PRINTCOLOR_END = '\033[0m'
# channel_shift_range=50,
emotion_model = choose_net(USE_EMOTION_MODEL, INPUT_SHAPE, EMOTION_NUM_CLS)
sgd = optimizers.SGD(lr=LEARNING_RATE, decay=LEARNING_RATE/BATCH_SIZE, momentum=0.9, nesterov=True)
emotion_model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
# callbacks
csv_logger = CSVLogger(EMOTION_LOG_NAME, append=False)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE/4), verbose=1)
# model_names = trained_models_path + '.{epoch:02d}-{val_acc:.2f}.hdf5'
model_checkpoint = ModelCheckpoint(EMOTION_MODEL_NAME, 'val_loss', verbose=1,
save_weights_only=False, save_best_only=True)
callbacks = [model_checkpoint, csv_logger, reduce_lr, early_stop]
# loading dataset
data_loader = DataManager(USE_EMOTION_DATASET, image_size=INPUT_SHAPE[:2])
faces, emotions, usages = data_loader.get_data()
faces = process_img(faces)
num_samples, num_classes = emotions.shape
train_data, val_data = split_raf_data(faces, emotions, usages)
train_faces, train_emotions = train_data
# if os.path.exists(EMOTION_MODEL_NAME):
# emotion_net = load_model(EMOTION_MODEL_NAME)
emotion_model.fit_generator(data_generator.flow(train_faces, train_emotions, BATCH_SIZE),
steps_per_epoch=len(train_faces) / BATCH_SIZE,epochs=EPOCHS,
verbose=1, callbacks=callbacks, validation_data=val_data)
if IS_CONVERT2TFLITE:
n_lines_count.append(song.number_of_lines)
n_words_count.append(song.number_of_words)
n_chars_count.append(song.number_of_chars)
lines_counter = 0
for song in test_song_entries:
song.start_index = lines_counter
lines_counter += song.number_of_lines
n_lines_count.append(song.number_of_lines)
n_words_count.append(song.number_of_words)
n_chars_count.append(song.number_of_chars)
n_lines_count = np.asarray(n_lines_count)
n_words_count = np.asarray(n_words_count)
n_chars_count = np.asarray(n_chars_count)
print('Average number of lines ', n_lines_count.mean())
print('Average number of words ', n_words_count.mean())
print('Average number of chars ', n_chars_count.mean())
data_manager = DataManager(main_path)
data_manager.save_python_obj(train_song_entries, 'song_lyrics.train')
data_manager.save_python_obj(validation_song_entries, 'song_lyrics.validation')
data_manager.save_python_obj(test_song_entries, 'song_lyrics.test')
save_dataset_text(train_song_entries, embeddings_folder_path,
'embeddings.train.txt')
save_dataset_text(validation_song_entries, embeddings_folder_path,
'embeddings.validation.txt')
save_dataset_text(test_song_entries, embeddings_folder_path,
'embeddings.test.txt')
def setup(self, data_manager: DataManager, set_name: str) -> List[Song]:
x: List[Song] = data_manager.load_python_obj(f'song_lyrics.{set_name}')
return x
kIOnly = False
# ! Automatic Generated
kH5ModuleDataPath = os.path.join(kH5DataPath, 'h5_module_data')
kH5TrainTestDataPath = os.path.join(kH5DataPath, 'h5_train_test_split')
kLogPath = os.path.join('.', 'log', 'tf.' + os.path.split(kH5DataPath)[1] + f'.LSTM.{kLogPathComment}.log')
kSnapshotPath = os.path.join(kLogPath, 'snapshot', 'LSTM')
kRecoverMetaFile = kSnapshotPath + '-{}.meta'.format(kRecoverEpochNum)
kRecoverDataFile = kSnapshotPath + '-{}'.format(kRecoverEpochNum)
if __name__ == '__main__':
# data and log manager
data_manager = DataManager(kH5TrainTestDataPath, kH5ModuleDataPath, I_only=kIOnly, down_sample=0)
logger = Logger(os.path.join(kLogPath, 'lstm_train_val.log')).logger
# build model
lstm_model = BuildModel(data_manager.classes_num, num_hidden=kHiddenStateNum, I_only=kIOnly)
lstm_model.build()
loss = lstm_model.loss()
tf.summary.scalar('loss', loss)
optimizer = lstm_model.optimizer(loss, kLearningRate)
accuracy = lstm_model.accuracy()
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
parser.add_argument('--lang',
type=str,
default='en',
choices=['en', 'ko', 'ja', 'roman', 'mayan'])
parser.add_argument('--hidden', type=int, default=32, choices=[16, 32])
parser.add_argument('--lstm-layers', type=int, default=1, choices=[1, 2])
parser.add_argument('--use-nalu', type=int, default=1, choices=[0, 1])
parser.add_argument('--lr', type=float, default=1e-2, choices=[1e-2, 1e-3])
parser.add_argument('--reduce-sum', type=int, default=0, choices=[0, 1])
parser.add_argument('--reduce-lr', type=int, default=1, choices=[0, 1])
parser.add_argument('--epochs', type=int, default=591)
parser.add_argument('--batch-size', type=int, default=30)
args = parser.parse_args()
Num = __import__('lang.num.%s' % args.lang, fromlist=['Num'])
data_manager = DataManager(getattr(Num, 'Num'))
# TODO: Support more 'divide' option - now only English available
data_train, data_val, data_eval = data_manager.divide(
data_manager.generate(),
include=([
*range(20),
*range(20, 101, 10),
123,
], [], []))
model = Model(len(data_manager), args.hidden, args.lstm_layers,
args.use_nalu, args.reduce_sum)
op = optim.Adam(model.parameters(), lr=args.lr)
trainer = TrainManager(model, op, data_manager, args.reduce_lr, data_train,
data_val, data_eval)