def train_model(model: BaseModel, train_data: DataFrame, dev_data: DataFrame):
model_name = model.__class__.__name__
train_time = time.localtime()
add_log_file(logger, "log/%s_%s.log" % (model_name, time.strftime("%Y%m%d%H%M%S", train_time)))
logger.info("Training %s..." % model_name)
config: BaseConfig = model.config
logger.info(config.__dict__)
model.to(config.device)
opt = torch.optim.Adam(model.parameters(), config.learning_rate, weight_decay=config.l2_regularization)
lr_s = lr_scheduler.ExponentialLR(opt, gamma=config.learning_rate_decay)
loss = torch.nn.MSELoss()
last_progress = 0.
last_loss = float("inf")
train_data_iter = get_data_loader(train_data, config)
dev_data_iter = get_data_loader(dev_data, config)
batches_num = math.ceil(len(train_data) / float(config.batch_size))
while model.current_epoch < config.num_epochs:
model.train()
for batch_id, iter_i in enumerate(train_data_iter):
user_review, item_review, rating = iter_i
user_review = user_review.to(config.device)
item_review = item_review.to(config.device)
rating = rating.to(config.device)
opt.zero_grad()
predict = model(user_review, item_review)
li = loss(predict, rating)
li.backward()
opt.step()
# log progress
current_batches = model.current_epoch * batches_num + (batch_id + 1.0)
total_batches = config.num_epochs * batches_num
progress = current_batches / total_batches
if progress - last_progress > 0.001:
logger.debug("epoch %d, batch %d, loss: %f (%.2f%%)" %
(model.current_epoch, batch_id, li.item(), 100.0 * progress))
last_progress = progress
# complete one epoch
train_loss = eval_model(model, train_data_iter, loss)
dev_loss = eval_model(model, dev_data_iter, loss)
logger.info("Epoch %d complete. Total loss(train/dev)=%f/%f"
% (model.current_epoch, train_loss, dev_loss))
# save best model
if train_loss < last_loss:
last_loss = train_loss
save_model(model, train_time)
lr_s.step(model.current_epoch)
model.current_epoch += 1
logger.info("%s trained!" % model_name)
remove_log_file(logger)
def get_object(self, params):
query_str = BaseModel.get_query("tabla_get_object")
result_set = BaseModel.execute_query(query_str, params)
obj = DomainTabla()
if len(result_set) == 1:
for element_dict in result_set:
BaseModel.dict_to_object(obj, element_dict)
return obj
def insertar(self, obj=None):
stmt = BaseModel.get_query("tipo_dato_insert")
list_data = list(obj.__dict__.values())
list_data.pop(0)
insert_data = tuple(list_data)
obj.tipo_dato_id = self.execute_insert(stmt, insert_data)
def insertar(self, obj=None):
stmt = BaseModel.get_query("proveedor_bd_insert")
"""now = datetime.datetime.now()
obj.fch_creacion = now
fch_creacion = str(now.strftime("%Y-%m-%d"))"""
insert_data = (
obj.nombre,
obj.icono
)
obj.proveedor_bd_id = self.execute_insert(stmt, insert_data)
def eval_model(model: BaseModel, data_iter: DataLoader, loss) -> float:
model.eval()
model_name = model.__class__.__name__
config: BaseConfig = model.config
logger.debug("Evaluating %s..." % model_name)
with torch.no_grad():
predicts = []
ratings = []
for batch_id, iter_i in enumerate(data_iter):
user_review, item_review, rating = iter_i
user_review = user_review.to(config.device)
item_review = item_review.to(config.device)
rating = rating.to(config.device)
predict = model(user_review, item_review)
predicts.append(predict)
ratings.append(rating)
predicts = torch.cat(predicts)
ratings = torch.cat(ratings)
return loss(predicts, ratings).item()
def insertar(self, obj=None):
stmt = BaseModel.get_query("tabla_insert")
now = datetime.datetime.now()
#Convert to iso format to insert
#obj.fch_registro = str(now.strftime("%Y-%m-%d"))
obj.fch_creacion = now
fch_creacion = str(now.strftime("%Y-%m-%d"))
insert_data = (obj.nombre, obj.descripcion, fch_creacion, obj.dbms_id)
obj.tabla_id = self.execute_insert(stmt, insert_data)
def insertar(self, obj=None):
stmt = BaseModel.get_query("campo_insert")
now = datetime.datetime.now()
obj.fch_creacion = now
fch_creacion = str(now.strftime("%Y-%m-%d"))
insert_data = (obj.tabla_id, obj.nombre, obj.descripcion,
obj.flg_obligatorio, obj.flg_pk, fch_creacion,
obj.tipo_dato_id, obj.tipo_dato_data,
obj.tipo_dato_text, obj.tipo_dato_syntax)
obj.campo_id = self.execute_insert(stmt, insert_data)
def actualizar(self, obj=None):
stmt = BaseModel.get_query("tabla_actualizar")
self.execute_update(stmt, obj.__dict__)
def __init__(self):
self.model = BaseModel()