#save_lrs = []
#def lr_schedule(epoch, lr, **kwargs):
# """exponential decay"""
# new_lr = lr[0] * 1e-5**(epoch / 200)
# save_lrs.append(new_lr)
# return new_lr
#callbacks = [LearningRateScheduler(lr_schedule)]
#import matplotlib.pyplot as plt
#%matplotlib inline
#plt.plot(np.arange(len(save_lrs)), np.array(save_lrs))
#plt.show()
from torchsample.callbacks import ReduceLROnPlateau
callbacks = [
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=1,
cooldown=0,
min_lr=1e-3,
verbose=1)
]
model = Network()
model.set_loss(F.nll_loss)
model.set_optimizer(optim.Adadelta, lr=1.0)
model.set_callbacks(callbacks)
# FIT THE MODEL
model.fit(x_train,
y_train,
validation_data=(x_test, y_test),
nb_epoch=20,
batch_size=128,
verbose=1)
def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2(x), 2))
x = x.view(-1, 1600)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x)
model = Network()
trainer = ModuleTrainer(model)
callbacks = [
EarlyStopping(patience=10),
ReduceLROnPlateau(factor=0.5, patience=5)
]
regularizers = [
L1Regularizer(scale=1e-3, module_filter='conv*'),
L2Regularizer(scale=1e-5, module_filter='fc*')
]
constraints = [UnitNorm(frequency=3, unit='batch', module_filter='fc*')]
initializers = [XavierUniform(bias=False, module_filter='fc*')]
metrics = [CategoricalAccuracy(top_k=3)]
trainer.compile(loss='nll_loss',
optimizer='adadelta',
regularizers=regularizers,
constraints=constraints,
initializers=initializers,
metrics=metrics)
valid_loader = DataLoader(
dataset,
batch_size=args.batch_size,
sampler=valid_sampler,
num_workers=4,
pin_memory=True if th.cuda.is_available() else False)
trainer = ModuleTrainer(model)
suffix = '-' + args.suffix if args.suffix else ''
if args.squeeze_excitation:
suffix = '-se' + suffix
callbacks = [
ReduceLROnPlateau(factor=0.5, patience=5, verbose=1),
ModelCheckpoint(directory='checkpoints',
filename='fishnet' + suffix + '_{epoch}_{loss}.pth',
save_best_only=True,
verbose=1)
]
initializers = [XavierUniform(bias=False, module_filter='class*')]
metrics = [CategoricalAccuracy()]
def species_length_loss(input, target):
input_species = input[:, :8]
input_length = input[:, 8]
target_species = target[:, 0].long()
def train(kwargs):
"""
Training Process
:return:
"""
print('[INFO] Loading Settings...')
parser = None
config = None
try:
parser = NeuralMFConfig()
# print(kwargs)
if '-c' in kwargs:
args = list(enumerate(kwargs))
for id, arg in args:
if '-c' == arg and len(args) >= id + 2:
config = parser.get_args_from_json(args[id + 1][1])
break
else:
raise AssertionError("Corresponding config arg not found")
else:
config = parser.parse_args(kwargs)
except Exception as e:
print('[Exception] Unavailable Settings, %s' % e)
if parser:
help(kwargs)
if '-c' in kwargs:
print(
'[Exception] Please refer formatting: python main.py -c configs/neuralMF_config.json'
)
exit(0)
cuda_device = -1 if not config.cuda else 0
if config.cuda:
torch.cuda.set_device(cuda_device)
config.cuda_device = cuda_device
print('[INFO] Loading Data...')
dl = CFDataLoader(config=config, only_test=False)
print('[INFO] Build Networks...')
nb_users, nb_items = dl.get_num_user_and_item()
parser.args.nb_users = nb_users
parser.args.nb_items = nb_items
model = implicit_load_model(config.model)(config, nb_users, nb_items)
print(model)
callbacks = [
EarlyStopping(patience=10),
ReduceLROnPlateau(factor=0.5, patience=5)
]
regularizers = [L2Regularizer(scale=1e-5, module_filter='fc*')]
constraints = [UnitNorm(frequency=3, unit='batch', module_filter='fc*')]
initializers = []
metrics = [HitAccuracy(config.topk), NDCGAccuracy(config.topk)]
print('[INFO] Begin Training...')
time_str = _time.strftime('%m%d_%H:%M:%S')
trainer = RankingModulelTrainer(model=model)
trainer.compile(loss="binary_cross_entropy_with_logits",
optimizer='Adam',
regularizers=regularizers,
constraints=constraints,
initializers=initializers,
metrics=metrics,
callbacks=callbacks)
trainer.fit_loader(dl.get_train_data(),
dl.get_test_data(),
num_epoch=config.epochs,
verbose=1,
cuda_device=cuda_device)
print('[INFO] Complete Training...')
model.save(time_str=time_str, use_onnx=config.onnx)
parser.save(timestamp=time_str)
print('[INFO] Saved Model into checkpoint directory')
def main(args):
"""Simply redirrcts to the correct function."""
start = default_timer()
args.cuda = not args.no_cuda and torch.cuda.is_available()
np.random.seed(args.seed)
torch.manual_seed(args.seed)
print("-------------------------------------------------")
if args.verbose > 0:
print("Ran on {}".format(time.strftime("%Y-%m-%d %H:%M")))
print()
print('Parameters: {}'.format(vars(args)))
print()
# PREPARES DATA
if args.verbose > 1:
print('Prepares data ...')
train, valid, test = train_valid_test_datasets(
args.dataset,
validSize=args.validation_size,
isHashingTrick=not args.dictionnary,
nFeaturesRange=args.num_features_range,
ngramRange=args.ngrams_range,
seed=args.seed,
num_words=args.num_embeding,
specificArgs={'dictionnary': ['num_words']})
num_classes = len(train.classes)
train = DataLoader(dataset=train,
batch_size=args.batch_size,
shuffle=not args.no_shuffle)
valid = DataLoader(dataset=valid,
batch_size=args.batch_size,
shuffle=not args.no_shuffle)
test = DataLoader(dataset=test,
batch_size=args.batch_size,
shuffle=not args.no_shuffle)
# PREPARES MODEL
if args.verbose > 1:
print('Prepares model ...')
Model = ModelNoDict if args.model == 'embed-softmax' else ModelDict
model = Model(args.num_embeding,
args.dim,
num_classes,
isHash=not args.no_hashembed,
seed=args.seed,
num_buckets=args.num_buckets,
append_weight=not args.no_append_weight,
aggregation_mode=args.agg_mode,
oldAlgorithm=args.old_hashembed)
if args.cuda:
model.cuda()
if args.verbose > 1:
model_parameters = filter(lambda p: p.requires_grad,
model.parameters())
nParams = sum([np.prod(p.size()) for p in model_parameters])
print('Num parameters in model: {}'.format(nParams))
print("Train on {} samples, validate on {} samples".format(
len(train), len(valid)))
# COMPILES
trainer = ModuleTrainer(model)
loss = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
callbacks = []
callbackMetric = "val_loss" if args.val_loss_callback else "val_acc_metric"
if args.patience is not None:
callbacks.append(
EarlyStopping(patience=args.patience, monitor=callbackMetric))
if args.plateau_reduce_lr is not None:
callbacks.append(
ReduceLROnPlateau(factor=args.plateau_reduce_lr[1],
patience=args.plateau_reduce_lr[0],
monitor=callbackMetric))
if not args.no_checkpoint:
modelDir = os.path.join(parentddir, 'models')
filename = "{}.pth.tar".format(args.dataset)
callbacks.append(
ModelCheckpoint(modelDir,
filename=filename,
save_best_only=True,
max_save=1,
monitor=callbackMetric))
metrics = [CategoricalAccuracy()]
trainer.compile(loss=loss,
optimizer=optimizer,
callbacks=callbacks,
metrics=metrics)
# TRAINS
if args.verbose > 1:
print('Trains ...')
trainer.fit_loader(train,
val_loader=valid,
num_epoch=args.epochs,
verbose=args.verbose,
cuda_device=0 if args.cuda else -1)
# EVALUATES
print()
evalTest = trainer.evaluate_loader(test,
verbose=args.verbose,
cuda_device=0 if args.cuda else -1)
evalValid = trainer.evaluate_loader(valid,
verbose=args.verbose,
cuda_device=0 if args.cuda else -1)
print("Last Model. Validation - Loss: {}, Accuracy: {}".format(
evalValid['val_loss'], evalValid['val_acc_metric']))
print("Last Model. Test - Loss: {}, Accuracy: {}".format(
evalTest['val_loss'], evalTest['val_acc_metric']))
if not args.no_checkpoint:
checkpoint = torch.load(os.path.join(modelDir, filename))
model.load_state_dict(checkpoint["state_dict"])
evalTest = trainer.evaluate_loader(test,
verbose=args.verbose,
cuda_device=0 if args.cuda else -1)
evalValid = trainer.evaluate_loader(valid,
verbose=args.verbose,
cuda_device=0 if args.cuda else -1)
print("Best Model. Validation - Loss: {}, Accuracy: {}".format(
evalValid['val_loss'], evalValid['val_acc_metric']))
print("Best Model. Test - Loss: {}, Accuracy: {}".format(
evalTest['val_loss'], evalTest['val_acc_metric']))
if args.verbose > 1:
print('Finished after {:.1f} min.'.format(
(default_timer() - start) / 60))