def run(option=None):
pl.trainer.seed_everything(40)
option = option or args.log
# set path
saved_dir = f"{args.mind_type}/{args.model_class}"
ckpt_dir = f"saved/checkpoint/{saved_dir}/{option}"
if args.resume and os.path.exists(args.resume):
best_model_path = args.resume
else:
best_model_path = os.path.join(ckpt_dir, "best_model.ckpt")
resume_path = best_model_path if os.path.exists(
best_model_path) and args.resume else None
train_news_file, train_behaviors_file = get_path("train",
mind_type=args.mind_type)
valid_news_file, valid_behaviors_file = get_path("valid",
mind_type=args.mind_type)
converter = Converter(hparams).converter
train_dataset = TrainingDataset(train_news_file,
train_behaviors_file,
hparams,
converter,
npratio=hparams.npratio)
train_dataloader = DataLoader(train_dataset,
hparams.batch_size,
num_workers=args.num_workers,
pin_memory=True)
hparams.update(**{"user_embedding_size": len(train_dataset.uid2index)})
# set validation interval and max epochs
interval, epochs = len(train_dataloader) // 3, hparams.epochs
accelerator = "ddp" if int(args.gpus) > 1 else None
valid_callback = ValidationCallback(valid_news_file, valid_behaviors_file,
hparams, converter, ckpt_dir, interval)
tb_logger = pl_loggers.TensorBoardLogger(f"saved/logs/{saved_dir}")
# trainer object, uses good defaults (auto-tensorboard, checkpoints, logs, and more)
trainer = pl.Trainer(gpus=int(args.gpus),
accelerator=accelerator,
max_epochs=epochs,
deterministic=True,
logger=tb_logger,
callbacks=[valid_callback],
resume_from_checkpoint=resume_path,
profiler="simple")
model_class = get_model_class(args.model_class)
trainer.fit(model_class(hparams), train_dataloader)
group_auc = [
float(file.split("==")[1].replace(".ckpt", ""))
for file in os.listdir(ckpt_dir) if "==" in file
]
best_auc = max(group_auc)
for file in os.scandir(ckpt_dir):
if "best_model" in file.name:
continue
auc = float(file.name.split("==")[1].replace(".ckpt", ""))
if auc < best_auc:
if os.path.exists(file.path):
os.remove(file.path)
import os
# set up configuration
from configuration import get_path, load_trainer, get_data_path
from utils import tools
import argparse
parse = argparse.ArgumentParser(description="NRMS Training process")
parse.add_argument("--log", "-l", dest="log_file", metavar="FILE", help="log file", default="log.txt")
parse.add_argument("--configure", "-c", dest="config", metavar="FILE", help="yaml file", default=r"nrms.yaml")
parse.add_argument("--device_id", "-d", dest="device_id", metavar="INT", default=0)
parse.add_argument("--model_class", "-m", dest="model_class", metavar="TEXT", default="nrms")
parse.add_argument("--mind_type", "-t", dest="mind_type", metavar="TEXT", default="small")
parse.add_argument("--resume", "-r", dest="resume", metavar="INT", help="whether resume or not", default=0)
args = parse.parse_args()
train_news_file, train_behaviors_file = get_path("train", mind_type=args.mind_type)
valid_news_file, valid_behaviors_file = get_path("valid", mind_type=args.mind_type)
config = {"yaml_name": args.config, "log_file": args.log_file, "device_id": int(args.device_id),
"model_class": args.model_class}
trainer = load_trainer(**config)
# save model
model_path = os.path.join(get_data_path(mind_type=args.mind_type), "checkpoint")
os.makedirs(model_path, exist_ok=True)
resume = int(args.resume)
resume_path = os.path.join(model_path, "best_model.pth")
if resume:
trainer.fit(train_news_file, train_behaviors_file, valid_news_file, valid_behaviors_file, model_dir=model_path,
resume_path=resume_path)
else:
trainer.fit(train_news_file, train_behaviors_file, valid_news_file, valid_behaviors_file, model_dir=model_path)
import os
import torch
from callbacks import TestCallback, ValidationCallback
from configuration import get_path, get_params, get_argument, get_model_class
from utils.helpers import Converter
args = get_argument()
hparams = get_params(args.config,
head_num=int(args.head_num),
head_dim=int(args.head_dim),
model=args.model_class)
print(hparams)
test_news_file, test_behaviors_file = get_path("test",
mind_type=args.mind_type)
converter = Converter(hparams).converter
saved_dir = f"{args.mind_type}/{args.model_class}"
ckpt_dir = f"saved/checkpoint/{saved_dir}/{args.log}"
pred_dir = f"saved/prediction/{saved_dir}/{args.log}"
best_model_path = os.path.join(ckpt_dir, "best_model.ckpt")
if args.resume and os.path.exists(args.resume):
best_model_path = args.resume
load_path = best_model_path
if not os.path.exists(load_path):
raise FileNotFoundError
test_methods = ["linear", "conv", "multi_head", "lstm", "bilstm", "empty"]
att_methods = ["layer_att", "head_att"]
hparams.update(
**{
"news_layer": test_methods[int(args.news_layer)],
"user_layer": test_methods[int(args.user_layer)],
def serve(filepath):
"""Serve the static files (resources)"""
return bridge_response(
'STATIC:%s:%s' %
(filepath, configuration.get_path(MODULE_NAME, DIR_STATIC)))
# -*- coding: UTF-8 -*- import os.path import configuration from app_constants import DIR_CONF MODULE_NAME = os.path.basename(os.path.dirname(__file__)) SETTINGS_FILENAME = '%s%s.ini' % (MODULE_NAME, os.path.isfile( configuration.get_path(DIR_CONF, '%s_custom.ini' % MODULE_NAME)) \ and '_custom' or '') SETTINGS_DB = configuration.get_config_string( SETTINGS_FILENAME, 'GENERAL', 'SETTINGS')
Created on Tue Jan 26 15:48:49 2020
@author: susan
Exports telemanagement (telegestao) data to a csv -> one sensor per file
"""
import mysql.connector
import pandas as pd
import sys
sys.path.append('../Functions')
import configuration
root = configuration.read_config()
path_init = configuration.get_path(root)
db_config = configuration.get_db(root)
wmes = configuration.get_wmes(root)
mydb = mysql.connector.connect(
host=db_config['host'],
user=db_config['user'],
passwd=db_config['pw']
)
print(mydb)
print("\nExport initiated")
cursor = mydb.cursor(buffered=True)
for wme in wmes:
# -*- coding: UTF-8 -*-
import os.path
import configuration
from app_constants import DIR_CONF
MODULE_NAME = os.path.basename(os.path.dirname(__file__))
SETTINGS_FILENAME = '%s%s.ini' % (MODULE_NAME, os.path.isfile(
configuration.get_path(DIR_CONF, '%s_custom.ini' % MODULE_NAME)) \
and '_custom' or '')
SETTINGS_DB = configuration.get_config_string(SETTINGS_FILENAME, 'GENERAL',
'SETTINGS')
def serve(filepath):
"""Serve the static files (resources)"""
return bridge_response('STATIC:%s:%s' % (filepath,
configuration.get_path(MODULE_NAME, DIR_STATIC)))
